2024-11-08 00:23:26,534 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-08 00:23:26,584 main DEBUG Took 0.046028 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-08 00:23:26,585 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-08 00:23:26,587 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-08 00:23:26,589 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-08 00:23:26,592 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 00:23:26,643 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-08 00:23:26,696 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 00:23:26,700 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 00:23:26,701 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 00:23:26,701 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 00:23:26,702 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 00:23:26,703 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 00:23:26,704 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 00:23:26,705 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 00:23:26,706 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 00:23:26,706 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 00:23:26,708 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 00:23:26,708 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 00:23:26,709 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 00:23:26,709 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 00:23:26,710 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 00:23:26,711 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 00:23:26,711 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 00:23:26,712 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 00:23:26,713 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 00:23:26,713 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 00:23:26,714 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 00:23:26,714 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 00:23:26,715 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 00:23:26,720 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 00:23:26,721 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 00:23:26,722 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-08 00:23:26,723 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 00:23:26,725 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-08 00:23:26,727 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-08 00:23:26,728 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-08 00:23:26,740 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-08 00:23:26,741 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-08 00:23:26,761 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-08 00:23:26,772 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-08 00:23:26,776 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-08 00:23:26,778 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-08 00:23:26,786 main DEBUG createAppenders(={Console}) 2024-11-08 00:23:26,787 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-08 00:23:26,788 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-08 00:23:26,788 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-08 00:23:26,789 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-08 00:23:26,790 main DEBUG OutputStream closed 2024-11-08 00:23:26,790 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-08 00:23:26,790 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-08 00:23:26,791 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-08 00:23:26,899 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-08 00:23:26,902 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-08 00:23:26,903 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-08 00:23:26,904 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-08 00:23:26,905 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-08 00:23:26,905 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-08 00:23:26,906 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-08 00:23:26,906 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-08 00:23:26,907 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-08 00:23:26,907 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-08 00:23:26,908 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-08 00:23:26,908 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-08 00:23:26,909 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-08 00:23:26,909 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-08 00:23:26,909 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-08 00:23:26,910 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-08 00:23:26,910 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-08 00:23:26,911 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-08 00:23:26,913 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-08 00:23:26,914 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-08 00:23:26,914 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-08 00:23:26,915 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-08T00:23:27,226 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c8e766d-da5e-64ca-5ccd-0c91fcfb6de3 2024-11-08 00:23:27,232 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-08 00:23:27,233 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-08T00:23:27,246 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-08T00:23:27,319 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-08T00:23:27,340 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c8e766d-da5e-64ca-5ccd-0c91fcfb6de3/cluster_72fb0791-5876-a833-2fa2-61645688b81f, deleteOnExit=true 2024-11-08T00:23:27,340 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-08T00:23:27,342 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c8e766d-da5e-64ca-5ccd-0c91fcfb6de3/test.cache.data in system properties and HBase conf 2024-11-08T00:23:27,343 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c8e766d-da5e-64ca-5ccd-0c91fcfb6de3/hadoop.tmp.dir in system properties and HBase conf 2024-11-08T00:23:27,344 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c8e766d-da5e-64ca-5ccd-0c91fcfb6de3/hadoop.log.dir in system properties and HBase conf 2024-11-08T00:23:27,345 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c8e766d-da5e-64ca-5ccd-0c91fcfb6de3/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-08T00:23:27,346 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c8e766d-da5e-64ca-5ccd-0c91fcfb6de3/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-08T00:23:27,347 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-08T00:23:27,514 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-08T00:23:27,680 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T00:23:27,686 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c8e766d-da5e-64ca-5ccd-0c91fcfb6de3/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-08T00:23:27,687 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c8e766d-da5e-64ca-5ccd-0c91fcfb6de3/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-08T00:23:27,688 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c8e766d-da5e-64ca-5ccd-0c91fcfb6de3/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-08T00:23:27,689 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c8e766d-da5e-64ca-5ccd-0c91fcfb6de3/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-08T00:23:27,690 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c8e766d-da5e-64ca-5ccd-0c91fcfb6de3/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-08T00:23:27,690 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c8e766d-da5e-64ca-5ccd-0c91fcfb6de3/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-08T00:23:27,691 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c8e766d-da5e-64ca-5ccd-0c91fcfb6de3/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-08T00:23:27,692 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c8e766d-da5e-64ca-5ccd-0c91fcfb6de3/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-08T00:23:27,692 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c8e766d-da5e-64ca-5ccd-0c91fcfb6de3/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-08T00:23:27,693 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c8e766d-da5e-64ca-5ccd-0c91fcfb6de3/nfs.dump.dir in system properties and HBase conf 2024-11-08T00:23:27,694 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c8e766d-da5e-64ca-5ccd-0c91fcfb6de3/java.io.tmpdir in system properties and HBase conf 2024-11-08T00:23:27,694 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c8e766d-da5e-64ca-5ccd-0c91fcfb6de3/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-08T00:23:27,695 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c8e766d-da5e-64ca-5ccd-0c91fcfb6de3/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-08T00:23:27,696 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c8e766d-da5e-64ca-5ccd-0c91fcfb6de3/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-08T00:23:29,221 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-08T00:23:29,305 INFO [Time-limited test {}] log.Log(170): Logging initialized @4150ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-08T00:23:29,383 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-08T00:23:29,453 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-08T00:23:29,481 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-08T00:23:29,482 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-08T00:23:29,484 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-08T00:23:29,501 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-08T00:23:29,505 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@610d6b27{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c8e766d-da5e-64ca-5ccd-0c91fcfb6de3/hadoop.log.dir/,AVAILABLE} 2024-11-08T00:23:29,506 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@31a6a643{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-08T00:23:30,270 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7f405bdd{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c8e766d-da5e-64ca-5ccd-0c91fcfb6de3/java.io.tmpdir/jetty-localhost-43653-hadoop-hdfs-3_4_1-tests_jar-_-any-3766650794567115718/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-08T00:23:30,296 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5b829fcb{HTTP/1.1, (http/1.1)}{localhost:43653} 2024-11-08T00:23:30,296 INFO [Time-limited test {}] server.Server(415): Started @5143ms 2024-11-08T00:23:30,928 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-08T00:23:30,937 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-08T00:23:30,938 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-08T00:23:30,938 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-08T00:23:30,939 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-08T00:23:30,940 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7a38c5d7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c8e766d-da5e-64ca-5ccd-0c91fcfb6de3/hadoop.log.dir/,AVAILABLE} 2024-11-08T00:23:30,940 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3815836d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-08T00:23:31,070 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@37e9dd02{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c8e766d-da5e-64ca-5ccd-0c91fcfb6de3/java.io.tmpdir/jetty-localhost-37933-hadoop-hdfs-3_4_1-tests_jar-_-any-18040758588637024081/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-08T00:23:31,071 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@8110c28{HTTP/1.1, (http/1.1)}{localhost:37933} 2024-11-08T00:23:31,071 INFO [Time-limited test {}] server.Server(415): Started @5918ms 2024-11-08T00:23:31,123 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-08T00:23:32,675 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c8e766d-da5e-64ca-5ccd-0c91fcfb6de3/cluster_72fb0791-5876-a833-2fa2-61645688b81f/data/data2/current/BP-1018477837-172.17.0.3-1731025408374/current, will proceed with Du for space computation calculation, 2024-11-08T00:23:32,675 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c8e766d-da5e-64ca-5ccd-0c91fcfb6de3/cluster_72fb0791-5876-a833-2fa2-61645688b81f/data/data1/current/BP-1018477837-172.17.0.3-1731025408374/current, will proceed with Du for space computation calculation, 2024-11-08T00:23:32,781 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-08T00:23:32,889 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xafa3746c98e4e190 with lease ID 0x9e9b84f4fdc834ce: Processing first storage report for DS-be635dc0-6437-4ba0-b479-97b2f6ad84a4 from datanode DatanodeRegistration(127.0.0.1:39671, datanodeUuid=6000dce3-f1e3-464a-a17d-199c856fc814, infoPort=35405, infoSecurePort=0, ipcPort=40979, storageInfo=lv=-57;cid=testClusterID;nsid=2118851093;c=1731025408374) 2024-11-08T00:23:32,891 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xafa3746c98e4e190 with lease ID 0x9e9b84f4fdc834ce: from storage DS-be635dc0-6437-4ba0-b479-97b2f6ad84a4 node DatanodeRegistration(127.0.0.1:39671, datanodeUuid=6000dce3-f1e3-464a-a17d-199c856fc814, infoPort=35405, infoSecurePort=0, ipcPort=40979, storageInfo=lv=-57;cid=testClusterID;nsid=2118851093;c=1731025408374), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-08T00:23:32,891 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xafa3746c98e4e190 with lease ID 0x9e9b84f4fdc834ce: Processing first storage report for DS-4f593737-6455-439b-93ac-4d4b56824e13 from datanode DatanodeRegistration(127.0.0.1:39671, datanodeUuid=6000dce3-f1e3-464a-a17d-199c856fc814, infoPort=35405, infoSecurePort=0, ipcPort=40979, storageInfo=lv=-57;cid=testClusterID;nsid=2118851093;c=1731025408374) 2024-11-08T00:23:32,892 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xafa3746c98e4e190 with lease ID 0x9e9b84f4fdc834ce: from storage DS-4f593737-6455-439b-93ac-4d4b56824e13 node DatanodeRegistration(127.0.0.1:39671, datanodeUuid=6000dce3-f1e3-464a-a17d-199c856fc814, infoPort=35405, infoSecurePort=0, ipcPort=40979, storageInfo=lv=-57;cid=testClusterID;nsid=2118851093;c=1731025408374), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-08T00:23:32,926 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c8e766d-da5e-64ca-5ccd-0c91fcfb6de3 2024-11-08T00:23:33,171 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c8e766d-da5e-64ca-5ccd-0c91fcfb6de3/cluster_72fb0791-5876-a833-2fa2-61645688b81f/zookeeper_0, clientPort=49210, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c8e766d-da5e-64ca-5ccd-0c91fcfb6de3/cluster_72fb0791-5876-a833-2fa2-61645688b81f/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c8e766d-da5e-64ca-5ccd-0c91fcfb6de3/cluster_72fb0791-5876-a833-2fa2-61645688b81f/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-08T00:23:33,188 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=49210 2024-11-08T00:23:33,206 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T00:23:33,212 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T00:23:33,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741825_1001 (size=7) 2024-11-08T00:23:34,078 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f with version=8 2024-11-08T00:23:34,078 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/hbase-staging 2024-11-08T00:23:34,221 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-08T00:23:34,580 INFO [Time-limited test {}] client.ConnectionUtils(128): master/3302f0f507bd:0 server-side Connection retries=45 2024-11-08T00:23:34,602 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T00:23:34,603 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T00:23:34,611 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T00:23:34,612 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T00:23:34,612 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T00:23:34,928 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-08T00:23:35,066 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-08T00:23:35,083 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-08T00:23:35,087 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T00:23:35,115 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 30857 (auto-detected) 2024-11-08T00:23:35,116 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-08T00:23:35,136 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:36785 2024-11-08T00:23:35,166 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36785 connecting to ZooKeeper ensemble=127.0.0.1:49210 2024-11-08T00:23:35,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:367850x0, quorum=127.0.0.1:49210, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T00:23:35,325 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36785-0x10117d39f8d0000 connected 2024-11-08T00:23:35,685 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T00:23:35,689 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T00:23:35,702 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36785-0x10117d39f8d0000, quorum=127.0.0.1:49210, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T00:23:35,708 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f, hbase.cluster.distributed=false 2024-11-08T00:23:35,743 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36785-0x10117d39f8d0000, quorum=127.0.0.1:49210, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T00:23:35,754 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36785 2024-11-08T00:23:35,759 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36785 2024-11-08T00:23:35,762 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36785 2024-11-08T00:23:35,768 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36785 2024-11-08T00:23:35,770 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36785 2024-11-08T00:23:35,897 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/3302f0f507bd:0 server-side Connection retries=45 2024-11-08T00:23:35,899 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T00:23:35,899 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T00:23:35,900 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T00:23:35,900 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T00:23:35,900 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T00:23:35,904 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-08T00:23:35,907 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T00:23:35,908 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:40479 2024-11-08T00:23:35,911 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40479 connecting to ZooKeeper ensemble=127.0.0.1:49210 2024-11-08T00:23:35,913 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T00:23:35,918 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T00:23:35,955 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:404790x0, quorum=127.0.0.1:49210, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T00:23:35,956 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:404790x0, quorum=127.0.0.1:49210, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T00:23:35,957 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40479-0x10117d39f8d0001 connected 2024-11-08T00:23:35,962 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-08T00:23:35,975 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-08T00:23:35,978 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40479-0x10117d39f8d0001, quorum=127.0.0.1:49210, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-08T00:23:35,987 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40479-0x10117d39f8d0001, quorum=127.0.0.1:49210, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T00:23:35,992 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40479 2024-11-08T00:23:35,995 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40479 2024-11-08T00:23:36,006 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40479 2024-11-08T00:23:36,012 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40479 2024-11-08T00:23:36,013 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40479 2024-11-08T00:23:36,043 DEBUG [M:0;3302f0f507bd:36785 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;3302f0f507bd:36785 2024-11-08T00:23:36,045 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/3302f0f507bd,36785,1731025414356 2024-11-08T00:23:36,062 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40479-0x10117d39f8d0001, quorum=127.0.0.1:49210, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T00:23:36,062 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36785-0x10117d39f8d0000, quorum=127.0.0.1:49210, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T00:23:36,066 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36785-0x10117d39f8d0000, quorum=127.0.0.1:49210, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/3302f0f507bd,36785,1731025414356 2024-11-08T00:23:36,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36785-0x10117d39f8d0000, quorum=127.0.0.1:49210, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:23:36,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40479-0x10117d39f8d0001, quorum=127.0.0.1:49210, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-08T00:23:36,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40479-0x10117d39f8d0001, quorum=127.0.0.1:49210, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:23:36,114 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36785-0x10117d39f8d0000, quorum=127.0.0.1:49210, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-08T00:23:36,116 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/3302f0f507bd,36785,1731025414356 from backup master directory 2024-11-08T00:23:36,125 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36785-0x10117d39f8d0000, quorum=127.0.0.1:49210, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/3302f0f507bd,36785,1731025414356 2024-11-08T00:23:36,125 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36785-0x10117d39f8d0000, quorum=127.0.0.1:49210, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T00:23:36,126 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40479-0x10117d39f8d0001, quorum=127.0.0.1:49210, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T00:23:36,128 WARN [master/3302f0f507bd:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T00:23:36,128 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=3302f0f507bd,36785,1731025414356 2024-11-08T00:23:36,131 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-08T00:23:36,144 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-08T00:23:36,227 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/hbase.id] with ID: 3187367b-61bb-4afc-a729-21b60de76def 2024-11-08T00:23:36,227 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/.tmp/hbase.id 2024-11-08T00:23:36,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741826_1002 (size=42) 2024-11-08T00:23:36,252 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/.tmp/hbase.id]:[hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/hbase.id] 2024-11-08T00:23:36,342 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T00:23:36,350 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-08T00:23:36,380 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 28ms. 2024-11-08T00:23:36,398 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36785-0x10117d39f8d0000, quorum=127.0.0.1:49210, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:23:36,398 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40479-0x10117d39f8d0001, quorum=127.0.0.1:49210, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:23:36,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741827_1003 (size=196) 2024-11-08T00:23:36,465 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T00:23:36,469 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-08T00:23:36,500 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:36,508 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T00:23:36,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741828_1004 (size=1189) 2024-11-08T00:23:36,986 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/MasterData/data/master/store 2024-11-08T00:23:37,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741829_1005 (size=34) 2024-11-08T00:23:37,419 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-08T00:23:37,424 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:23:37,425 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-08T00:23:37,426 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T00:23:37,426 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T00:23:37,428 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-08T00:23:37,429 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T00:23:37,429 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T00:23:37,431 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731025417425Disabling compacts and flushes for region at 1731025417425Disabling writes for close at 1731025417428 (+3 ms)Writing region close event to WAL at 1731025417429 (+1 ms)Closed at 1731025417429 2024-11-08T00:23:37,434 WARN [master/3302f0f507bd:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/MasterData/data/master/store/.initializing 2024-11-08T00:23:37,434 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/MasterData/WALs/3302f0f507bd,36785,1731025414356 2024-11-08T00:23:37,446 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T00:23:37,465 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3302f0f507bd%2C36785%2C1731025414356, suffix=, logDir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/MasterData/WALs/3302f0f507bd,36785,1731025414356, archiveDir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/MasterData/oldWALs, maxLogs=10 2024-11-08T00:23:37,488 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/MasterData/WALs/3302f0f507bd,36785,1731025414356/3302f0f507bd%2C36785%2C1731025414356.1731025417469, exclude list is [], retry=0 2024-11-08T00:23:37,505 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39671,DS-be635dc0-6437-4ba0-b479-97b2f6ad84a4,DISK] 2024-11-08T00:23:37,509 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-08T00:23:37,546 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/MasterData/WALs/3302f0f507bd,36785,1731025414356/3302f0f507bd%2C36785%2C1731025414356.1731025417469 2024-11-08T00:23:37,547 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35405:35405)] 2024-11-08T00:23:37,547 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-08T00:23:37,548 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:23:37,551 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T00:23:37,552 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T00:23:37,600 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T00:23:37,634 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-08T00:23:37,639 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:23:37,643 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T00:23:37,644 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T00:23:37,653 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-08T00:23:37,653 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:23:37,654 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:23:37,655 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T00:23:37,660 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-08T00:23:37,660 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:23:37,666 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:23:37,667 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T00:23:37,673 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-08T00:23:37,673 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:23:37,675 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:23:37,676 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T00:23:37,681 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-08T00:23:37,682 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-08T00:23:37,696 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T00:23:37,698 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T00:23:37,708 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T00:23:37,714 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T00:23:37,723 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T00:23:37,725 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60122229, jitterRate=-0.10410897433757782}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T00:23:37,737 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731025417564Initializing all the Stores at 1731025417567 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731025417568 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731025417569 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731025417570 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731025417570Cleaning up temporary data from old regions at 1731025417698 (+128 ms)Region opened successfully at 1731025417735 (+37 ms) 2024-11-08T00:23:37,739 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-08T00:23:37,810 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b439c11, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=3302f0f507bd/172.17.0.3:0 2024-11-08T00:23:37,862 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-08T00:23:37,878 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-08T00:23:37,878 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-08T00:23:37,883 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T00:23:37,885 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-08T00:23:37,890 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-08T00:23:37,891 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-08T00:23:37,934 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-08T00:23:37,948 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36785-0x10117d39f8d0000, quorum=127.0.0.1:49210, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-08T00:23:37,994 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-08T00:23:37,998 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-08T00:23:38,003 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36785-0x10117d39f8d0000, quorum=127.0.0.1:49210, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-08T00:23:38,041 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-08T00:23:38,045 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-08T00:23:38,051 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36785-0x10117d39f8d0000, quorum=127.0.0.1:49210, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-08T00:23:38,108 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-08T00:23:38,110 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36785-0x10117d39f8d0000, quorum=127.0.0.1:49210, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-08T00:23:38,173 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-08T00:23:38,198 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36785-0x10117d39f8d0000, quorum=127.0.0.1:49210, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-08T00:23:38,259 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-08T00:23:38,323 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40479-0x10117d39f8d0001, quorum=127.0.0.1:49210, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T00:23:38,323 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36785-0x10117d39f8d0000, quorum=127.0.0.1:49210, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T00:23:38,324 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40479-0x10117d39f8d0001, quorum=127.0.0.1:49210, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:23:38,324 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36785-0x10117d39f8d0000, quorum=127.0.0.1:49210, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:23:38,327 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=3302f0f507bd,36785,1731025414356, sessionid=0x10117d39f8d0000, setting cluster-up flag (Was=false) 2024-11-08T00:23:38,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36785-0x10117d39f8d0000, quorum=127.0.0.1:49210, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:23:38,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40479-0x10117d39f8d0001, quorum=127.0.0.1:49210, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:23:38,589 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-08T00:23:38,591 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=3302f0f507bd,36785,1731025414356 2024-11-08T00:23:38,727 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40479-0x10117d39f8d0001, quorum=127.0.0.1:49210, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:23:38,727 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36785-0x10117d39f8d0000, quorum=127.0.0.1:49210, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:23:38,840 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-08T00:23:38,845 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=3302f0f507bd,36785,1731025414356 2024-11-08T00:23:38,854 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-08T00:23:38,941 INFO [RS:0;3302f0f507bd:40479 {}] regionserver.HRegionServer(746): ClusterId : 3187367b-61bb-4afc-a729-21b60de76def 2024-11-08T00:23:38,945 DEBUG [RS:0;3302f0f507bd:40479 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-08T00:23:38,959 DEBUG [RS:0;3302f0f507bd:40479 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-08T00:23:38,959 DEBUG [RS:0;3302f0f507bd:40479 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-08T00:23:38,968 DEBUG [RS:0;3302f0f507bd:40479 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-08T00:23:38,969 DEBUG [RS:0;3302f0f507bd:40479 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4755102b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=3302f0f507bd/172.17.0.3:0 2024-11-08T00:23:38,974 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T00:23:38,985 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-08T00:23:38,991 DEBUG [RS:0;3302f0f507bd:40479 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;3302f0f507bd:40479 2024-11-08T00:23:38,994 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-08T00:23:38,995 INFO [RS:0;3302f0f507bd:40479 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-08T00:23:38,995 INFO [RS:0;3302f0f507bd:40479 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-08T00:23:38,995 DEBUG [RS:0;3302f0f507bd:40479 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-08T00:23:38,999 INFO [RS:0;3302f0f507bd:40479 {}] regionserver.HRegionServer(2659): reportForDuty to master=3302f0f507bd,36785,1731025414356 with port=40479, startcode=1731025415856 2024-11-08T00:23:39,010 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 3302f0f507bd,36785,1731025414356 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-08T00:23:39,016 DEBUG [RS:0;3302f0f507bd:40479 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-08T00:23:39,047 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/3302f0f507bd:0, corePoolSize=5, maxPoolSize=5 2024-11-08T00:23:39,047 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/3302f0f507bd:0, corePoolSize=5, maxPoolSize=5 2024-11-08T00:23:39,048 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/3302f0f507bd:0, corePoolSize=5, maxPoolSize=5 2024-11-08T00:23:39,051 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/3302f0f507bd:0, corePoolSize=5, maxPoolSize=5 2024-11-08T00:23:39,058 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/3302f0f507bd:0, corePoolSize=10, maxPoolSize=10 2024-11-08T00:23:39,061 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/3302f0f507bd:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:23:39,062 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/3302f0f507bd:0, corePoolSize=2, maxPoolSize=2 2024-11-08T00:23:39,062 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/3302f0f507bd:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:23:39,120 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48559, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-08T00:23:39,135 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36785 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:39,141 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T00:23:39,142 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-08T00:23:39,158 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731025449158 2024-11-08T00:23:39,160 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-08T00:23:39,162 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:23:39,162 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-08T00:23:39,162 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-08T00:23:39,167 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-08T00:23:39,167 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-08T00:23:39,168 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-08T00:23:39,168 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-08T00:23:39,180 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-08T00:23:39,231 DEBUG [RS:0;3302f0f507bd:40479 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-08T00:23:39,232 WARN [RS:0;3302f0f507bd:40479 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-08T00:23:39,233 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-08T00:23:39,234 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-08T00:23:39,235 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-08T00:23:39,240 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-08T00:23:39,241 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-08T00:23:39,244 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/3302f0f507bd:0:becomeActiveMaster-HFileCleaner.large.0-1731025419243,5,FailOnTimeoutGroup] 2024-11-08T00:23:39,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741831_1007 (size=1321) 2024-11-08T00:23:39,246 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/3302f0f507bd:0:becomeActiveMaster-HFileCleaner.small.0-1731025419244,5,FailOnTimeoutGroup] 2024-11-08T00:23:39,246 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-08T00:23:39,247 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-08T00:23:39,248 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-08T00:23:39,248 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-08T00:23:39,333 INFO [RS:0;3302f0f507bd:40479 {}] regionserver.HRegionServer(2659): reportForDuty to master=3302f0f507bd,36785,1731025414356 with port=40479, startcode=1731025415856 2024-11-08T00:23:39,336 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36785 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 3302f0f507bd,40479,1731025415856 2024-11-08T00:23:39,340 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36785 {}] master.ServerManager(517): Registering regionserver=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:39,359 DEBUG [RS:0;3302f0f507bd:40479 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f 2024-11-08T00:23:39,359 DEBUG [RS:0;3302f0f507bd:40479 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34031 2024-11-08T00:23:39,359 DEBUG [RS:0;3302f0f507bd:40479 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-08T00:23:39,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36785-0x10117d39f8d0000, quorum=127.0.0.1:49210, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T00:23:39,443 DEBUG [RS:0;3302f0f507bd:40479 {}] zookeeper.ZKUtil(111): regionserver:40479-0x10117d39f8d0001, quorum=127.0.0.1:49210, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/3302f0f507bd,40479,1731025415856 2024-11-08T00:23:39,444 WARN [RS:0;3302f0f507bd:40479 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T00:23:39,444 INFO [RS:0;3302f0f507bd:40479 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T00:23:39,444 DEBUG [RS:0;3302f0f507bd:40479 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/WALs/3302f0f507bd,40479,1731025415856 2024-11-08T00:23:39,452 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [3302f0f507bd,40479,1731025415856] 2024-11-08T00:23:39,511 INFO [RS:0;3302f0f507bd:40479 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-08T00:23:39,546 INFO [RS:0;3302f0f507bd:40479 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-08T00:23:39,553 INFO [RS:0;3302f0f507bd:40479 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-08T00:23:39,553 INFO [RS:0;3302f0f507bd:40479 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T00:23:39,557 INFO [RS:0;3302f0f507bd:40479 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-08T00:23:39,568 INFO [RS:0;3302f0f507bd:40479 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-08T00:23:39,570 INFO [RS:0;3302f0f507bd:40479 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-08T00:23:39,570 DEBUG [RS:0;3302f0f507bd:40479 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/3302f0f507bd:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:23:39,571 DEBUG [RS:0;3302f0f507bd:40479 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/3302f0f507bd:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:23:39,571 DEBUG [RS:0;3302f0f507bd:40479 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/3302f0f507bd:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:23:39,576 DEBUG [RS:0;3302f0f507bd:40479 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/3302f0f507bd:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:23:39,577 DEBUG [RS:0;3302f0f507bd:40479 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/3302f0f507bd:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:23:39,577 DEBUG [RS:0;3302f0f507bd:40479 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/3302f0f507bd:0, corePoolSize=2, maxPoolSize=2 2024-11-08T00:23:39,578 DEBUG [RS:0;3302f0f507bd:40479 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:23:39,578 DEBUG [RS:0;3302f0f507bd:40479 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/3302f0f507bd:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:23:39,579 DEBUG [RS:0;3302f0f507bd:40479 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/3302f0f507bd:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:23:39,579 DEBUG [RS:0;3302f0f507bd:40479 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/3302f0f507bd:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:23:39,580 DEBUG [RS:0;3302f0f507bd:40479 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/3302f0f507bd:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:23:39,580 DEBUG [RS:0;3302f0f507bd:40479 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/3302f0f507bd:0, corePoolSize=1, maxPoolSize=1 2024-11-08T00:23:39,581 DEBUG [RS:0;3302f0f507bd:40479 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/3302f0f507bd:0, corePoolSize=3, maxPoolSize=3 2024-11-08T00:23:39,581 DEBUG [RS:0;3302f0f507bd:40479 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0, corePoolSize=3, maxPoolSize=3 2024-11-08T00:23:39,583 INFO [RS:0;3302f0f507bd:40479 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T00:23:39,583 INFO [RS:0;3302f0f507bd:40479 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T00:23:39,584 INFO [RS:0;3302f0f507bd:40479 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T00:23:39,584 INFO [RS:0;3302f0f507bd:40479 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-08T00:23:39,584 INFO [RS:0;3302f0f507bd:40479 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-08T00:23:39,584 INFO [RS:0;3302f0f507bd:40479 {}] hbase.ChoreService(168): Chore ScheduledChore name=3302f0f507bd,40479,1731025415856-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T00:23:39,651 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-08T00:23:39,651 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f 2024-11-08T00:23:39,670 INFO [RS:0;3302f0f507bd:40479 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-08T00:23:39,672 INFO [RS:0;3302f0f507bd:40479 {}] hbase.ChoreService(168): Chore ScheduledChore name=3302f0f507bd,40479,1731025415856-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T00:23:39,672 INFO [RS:0;3302f0f507bd:40479 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T00:23:39,673 INFO [RS:0;3302f0f507bd:40479 {}] regionserver.Replication(171): 3302f0f507bd,40479,1731025415856 started 2024-11-08T00:23:39,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741832_1008 (size=32) 2024-11-08T00:23:39,710 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:23:39,714 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-08T00:23:39,717 INFO [RS:0;3302f0f507bd:40479 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T00:23:39,717 INFO [RS:0;3302f0f507bd:40479 {}] regionserver.HRegionServer(1482): Serving as 3302f0f507bd,40479,1731025415856, RpcServer on 3302f0f507bd/172.17.0.3:40479, sessionid=0x10117d39f8d0001 2024-11-08T00:23:39,718 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-08T00:23:39,718 DEBUG [RS:0;3302f0f507bd:40479 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-08T00:23:39,718 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:23:39,719 DEBUG [RS:0;3302f0f507bd:40479 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 3302f0f507bd,40479,1731025415856 2024-11-08T00:23:39,719 DEBUG [RS:0;3302f0f507bd:40479 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3302f0f507bd,40479,1731025415856' 2024-11-08T00:23:39,719 DEBUG [RS:0;3302f0f507bd:40479 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-08T00:23:39,720 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T00:23:39,720 DEBUG [RS:0;3302f0f507bd:40479 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-08T00:23:39,720 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-08T00:23:39,721 DEBUG [RS:0;3302f0f507bd:40479 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-08T00:23:39,722 DEBUG [RS:0;3302f0f507bd:40479 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-08T00:23:39,722 DEBUG [RS:0;3302f0f507bd:40479 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 3302f0f507bd,40479,1731025415856 2024-11-08T00:23:39,722 DEBUG [RS:0;3302f0f507bd:40479 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3302f0f507bd,40479,1731025415856' 2024-11-08T00:23:39,722 DEBUG [RS:0;3302f0f507bd:40479 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-08T00:23:39,723 DEBUG [RS:0;3302f0f507bd:40479 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-08T00:23:39,724 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-08T00:23:39,725 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:23:39,725 DEBUG [RS:0;3302f0f507bd:40479 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-08T00:23:39,725 INFO [RS:0;3302f0f507bd:40479 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-08T00:23:39,725 INFO [RS:0;3302f0f507bd:40479 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-08T00:23:39,726 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T00:23:39,726 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-08T00:23:39,730 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-08T00:23:39,730 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:23:39,731 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T00:23:39,732 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-08T00:23:39,736 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-08T00:23:39,736 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:23:39,737 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T00:23:39,737 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-08T00:23:39,739 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/hbase/meta/1588230740 2024-11-08T00:23:39,740 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/hbase/meta/1588230740 2024-11-08T00:23:39,745 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-08T00:23:39,745 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-08T00:23:39,747 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-08T00:23:39,751 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-08T00:23:39,763 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T00:23:39,765 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62534771, jitterRate=-0.06815929710865021}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-08T00:23:39,768 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731025419710Initializing all the Stores at 1731025419713 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731025419713Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731025419714 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731025419714Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731025419714Cleaning up temporary data from old regions at 1731025419745 (+31 ms)Region opened successfully at 1731025419768 (+23 ms) 2024-11-08T00:23:39,768 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-08T00:23:39,769 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-08T00:23:39,769 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-08T00:23:39,769 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-08T00:23:39,769 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-08T00:23:39,772 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-08T00:23:39,772 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731025419768Disabling compacts and flushes for region at 1731025419768Disabling writes for close at 1731025419769 (+1 ms)Writing region close event to WAL at 1731025419771 (+2 ms)Closed at 1731025419772 (+1 ms) 2024-11-08T00:23:39,777 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T00:23:39,778 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-08T00:23:39,786 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-08T00:23:39,799 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T00:23:39,806 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-08T00:23:39,832 INFO [RS:0;3302f0f507bd:40479 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T00:23:39,837 INFO [RS:0;3302f0f507bd:40479 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3302f0f507bd%2C40479%2C1731025415856, suffix=, logDir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/WALs/3302f0f507bd,40479,1731025415856, archiveDir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/oldWALs, maxLogs=32 2024-11-08T00:23:39,862 DEBUG [RS:0;3302f0f507bd:40479 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/WALs/3302f0f507bd,40479,1731025415856/3302f0f507bd%2C40479%2C1731025415856.1731025419840, exclude list is [], retry=0 2024-11-08T00:23:39,870 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39671,DS-be635dc0-6437-4ba0-b479-97b2f6ad84a4,DISK] 2024-11-08T00:23:39,877 INFO [RS:0;3302f0f507bd:40479 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/WALs/3302f0f507bd,40479,1731025415856/3302f0f507bd%2C40479%2C1731025415856.1731025419840 2024-11-08T00:23:39,878 DEBUG [RS:0;3302f0f507bd:40479 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35405:35405)] 2024-11-08T00:23:39,959 DEBUG [3302f0f507bd:36785 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-08T00:23:39,970 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:39,978 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3302f0f507bd,40479,1731025415856, state=OPENING 2024-11-08T00:23:40,029 DEBUG [PEWorker-2 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-08T00:23:40,040 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36785-0x10117d39f8d0000, quorum=127.0.0.1:49210, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:23:40,040 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40479-0x10117d39f8d0001, quorum=127.0.0.1:49210, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:23:40,041 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:23:40,041 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:23:40,042 DEBUG [PEWorker-2 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T00:23:40,044 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=3302f0f507bd,40479,1731025415856}] 2024-11-08T00:23:40,220 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-08T00:23:40,224 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35313, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-08T00:23:40,243 INFO [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-08T00:23:40,243 INFO [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T00:23:40,244 INFO [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-08T00:23:40,249 INFO [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3302f0f507bd%2C40479%2C1731025415856.meta, suffix=.meta, logDir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/WALs/3302f0f507bd,40479,1731025415856, archiveDir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/oldWALs, maxLogs=32 2024-11-08T00:23:40,273 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/WALs/3302f0f507bd,40479,1731025415856/3302f0f507bd%2C40479%2C1731025415856.meta.1731025420252.meta, exclude list is [], retry=0 2024-11-08T00:23:40,278 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39671,DS-be635dc0-6437-4ba0-b479-97b2f6ad84a4,DISK] 2024-11-08T00:23:40,283 INFO [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/WALs/3302f0f507bd,40479,1731025415856/3302f0f507bd%2C40479%2C1731025415856.meta.1731025420252.meta 2024-11-08T00:23:40,284 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35405:35405)] 2024-11-08T00:23:40,284 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-08T00:23:40,287 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-08T00:23:40,291 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-08T00:23:40,297 INFO [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-08T00:23:40,304 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-08T00:23:40,305 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:23:40,305 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-08T00:23:40,306 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-08T00:23:40,320 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-08T00:23:40,323 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-08T00:23:40,324 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:23:40,325 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T00:23:40,336 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-08T00:23:40,339 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-08T00:23:40,339 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:23:40,341 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T00:23:40,342 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-08T00:23:40,344 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-08T00:23:40,344 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:23:40,346 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T00:23:40,346 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-08T00:23:40,349 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-08T00:23:40,349 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:23:40,351 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T00:23:40,352 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-08T00:23:40,357 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/hbase/meta/1588230740 2024-11-08T00:23:40,361 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/hbase/meta/1588230740 2024-11-08T00:23:40,365 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-08T00:23:40,366 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-08T00:23:40,367 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-08T00:23:40,372 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-08T00:23:40,375 INFO [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60264536, jitterRate=-0.10198843479156494}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-08T00:23:40,375 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-08T00:23:40,377 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731025420306Writing region info on filesystem at 1731025420307 (+1 ms)Initializing all the Stores at 1731025420310 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731025420310Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731025420310Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731025420310Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731025420311 (+1 ms)Cleaning up temporary data from old regions at 1731025420366 (+55 ms)Running coprocessor post-open hooks at 1731025420375 (+9 ms)Region opened successfully at 1731025420377 (+2 ms) 2024-11-08T00:23:40,387 INFO [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731025420211 2024-11-08T00:23:40,403 DEBUG [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-08T00:23:40,403 INFO [RS_OPEN_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-08T00:23:40,406 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:40,410 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3302f0f507bd,40479,1731025415856, state=OPEN 2024-11-08T00:23:40,463 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36785-0x10117d39f8d0000, quorum=127.0.0.1:49210, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:23:40,463 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40479-0x10117d39f8d0001, quorum=127.0.0.1:49210, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T00:23:40,464 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:23:40,464 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T00:23:40,464 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:40,472 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-08T00:23:40,472 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3302f0f507bd,40479,1731025415856 in 420 msec 2024-11-08T00:23:40,494 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-08T00:23:40,494 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 689 msec 2024-11-08T00:23:40,496 DEBUG [PEWorker-3 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T00:23:40,496 INFO [PEWorker-3 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-08T00:23:40,523 DEBUG [PEWorker-3 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:23:40,525 DEBUG [PEWorker-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:23:40,562 DEBUG [PEWorker-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:23:40,566 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52267, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:23:40,600 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6960 sec 2024-11-08T00:23:40,601 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731025420601, completionTime=-1 2024-11-08T00:23:40,604 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-08T00:23:40,605 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-08T00:23:40,656 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-08T00:23:40,656 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731025480656 2024-11-08T00:23:40,656 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731025540656 2024-11-08T00:23:40,656 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 51 msec 2024-11-08T00:23:40,659 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3302f0f507bd,36785,1731025414356-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T00:23:40,660 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3302f0f507bd,36785,1731025414356-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T00:23:40,660 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3302f0f507bd,36785,1731025414356-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T00:23:40,662 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-3302f0f507bd:36785, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T00:23:40,663 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-08T00:23:40,663 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-08T00:23:40,677 DEBUG [master/3302f0f507bd:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-08T00:23:40,713 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.584sec 2024-11-08T00:23:40,715 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-08T00:23:40,717 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-08T00:23:40,718 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-08T00:23:40,719 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-08T00:23:40,719 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-08T00:23:40,722 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3302f0f507bd,36785,1731025414356-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T00:23:40,723 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3302f0f507bd,36785,1731025414356-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-08T00:23:40,737 DEBUG [master/3302f0f507bd:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-08T00:23:40,738 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-08T00:23:40,739 INFO [master/3302f0f507bd:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3302f0f507bd,36785,1731025414356-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T00:23:40,759 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76f389d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:23:40,762 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-08T00:23:40,762 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-08T00:23:40,766 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:23:40,769 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:23:40,781 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:23:40,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:23:40,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:23:40,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c9ccd04, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:23:40,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:23:40,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:23:40,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:23:40,803 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56442, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:23:40,808 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d885a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:23:40,809 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:23:40,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:23:40,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:23:40,828 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53722, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:23:40,884 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=3302f0f507bd,36785,1731025414356 2024-11-08T00:23:40,913 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=598, ProcessCount=11, AvailableMemoryMB=5092 2024-11-08T00:23:40,929 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-08T00:23:40,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 3302f0f507bd,36785,1731025414356 2024-11-08T00:23:40,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5c63a26 2024-11-08T00:23:40,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-08T00:23:40,960 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56450, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-08T00:23:40,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-08T00:23:40,969 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T00:23:40,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-08T00:23:40,984 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T00:23:40,987 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-08T00:23:40,987 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:23:40,992 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T00:23:40,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T00:23:41,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741835_1011 (size=959) 2024-11-08T00:23:41,027 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f 2024-11-08T00:23:41,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741836_1012 (size=53) 2024-11-08T00:23:41,040 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:23:41,040 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing b211e3aadde87eeeb10e4f0dad149691, disabling compactions & flushes 2024-11-08T00:23:41,040 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:41,040 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:41,040 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. after waiting 0 ms 2024-11-08T00:23:41,040 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:41,040 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:41,040 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for b211e3aadde87eeeb10e4f0dad149691: Waiting for close lock at 1731025421040Disabling compacts and flushes for region at 1731025421040Disabling writes for close at 1731025421040Writing region close event to WAL at 1731025421040Closed at 1731025421040 2024-11-08T00:23:41,044 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T00:23:41,050 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731025421044"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731025421044"}]},"ts":"1731025421044"} 2024-11-08T00:23:41,056 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-08T00:23:41,059 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T00:23:41,062 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731025421059"}]},"ts":"1731025421059"} 2024-11-08T00:23:41,069 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-08T00:23:41,072 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b211e3aadde87eeeb10e4f0dad149691, ASSIGN}] 2024-11-08T00:23:41,077 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b211e3aadde87eeeb10e4f0dad149691, ASSIGN 2024-11-08T00:23:41,080 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b211e3aadde87eeeb10e4f0dad149691, ASSIGN; state=OFFLINE, location=3302f0f507bd,40479,1731025415856; forceNewPlan=false, retain=false 2024-11-08T00:23:41,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T00:23:41,233 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=b211e3aadde87eeeb10e4f0dad149691, regionState=OPENING, regionLocation=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:41,237 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b211e3aadde87eeeb10e4f0dad149691, ASSIGN because future has completed 2024-11-08T00:23:41,238 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856}] 2024-11-08T00:23:41,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T00:23:41,406 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:41,406 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} 2024-11-08T00:23:41,407 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b211e3aadde87eeeb10e4f0dad149691 2024-11-08T00:23:41,407 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:23:41,407 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for b211e3aadde87eeeb10e4f0dad149691 2024-11-08T00:23:41,407 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for b211e3aadde87eeeb10e4f0dad149691 2024-11-08T00:23:41,412 INFO [StoreOpener-b211e3aadde87eeeb10e4f0dad149691-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b211e3aadde87eeeb10e4f0dad149691 2024-11-08T00:23:41,416 INFO [StoreOpener-b211e3aadde87eeeb10e4f0dad149691-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b211e3aadde87eeeb10e4f0dad149691 columnFamilyName A 2024-11-08T00:23:41,416 DEBUG [StoreOpener-b211e3aadde87eeeb10e4f0dad149691-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:23:41,418 INFO [StoreOpener-b211e3aadde87eeeb10e4f0dad149691-1 {}] regionserver.HStore(327): Store=b211e3aadde87eeeb10e4f0dad149691/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:23:41,418 INFO [StoreOpener-b211e3aadde87eeeb10e4f0dad149691-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b211e3aadde87eeeb10e4f0dad149691 2024-11-08T00:23:41,422 INFO [StoreOpener-b211e3aadde87eeeb10e4f0dad149691-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b211e3aadde87eeeb10e4f0dad149691 columnFamilyName B 2024-11-08T00:23:41,422 DEBUG [StoreOpener-b211e3aadde87eeeb10e4f0dad149691-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:23:41,423 INFO [StoreOpener-b211e3aadde87eeeb10e4f0dad149691-1 {}] regionserver.HStore(327): Store=b211e3aadde87eeeb10e4f0dad149691/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:23:41,423 INFO [StoreOpener-b211e3aadde87eeeb10e4f0dad149691-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b211e3aadde87eeeb10e4f0dad149691 2024-11-08T00:23:41,427 INFO [StoreOpener-b211e3aadde87eeeb10e4f0dad149691-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b211e3aadde87eeeb10e4f0dad149691 columnFamilyName C 2024-11-08T00:23:41,427 DEBUG [StoreOpener-b211e3aadde87eeeb10e4f0dad149691-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:23:41,429 INFO [StoreOpener-b211e3aadde87eeeb10e4f0dad149691-1 {}] regionserver.HStore(327): Store=b211e3aadde87eeeb10e4f0dad149691/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:23:41,429 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for b211e3aadde87eeeb10e4f0dad149691 2024-11-08T00:23:41,432 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691 2024-11-08T00:23:41,433 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691 2024-11-08T00:23:41,436 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for b211e3aadde87eeeb10e4f0dad149691 2024-11-08T00:23:41,436 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for b211e3aadde87eeeb10e4f0dad149691 2024-11-08T00:23:41,437 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-08T00:23:41,440 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for b211e3aadde87eeeb10e4f0dad149691 2024-11-08T00:23:41,445 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T00:23:41,446 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened b211e3aadde87eeeb10e4f0dad149691; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63808328, jitterRate=-0.04918181896209717}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-08T00:23:41,447 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b211e3aadde87eeeb10e4f0dad149691 2024-11-08T00:23:41,447 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for b211e3aadde87eeeb10e4f0dad149691: Running coprocessor pre-open hook at 1731025421407Writing region info on filesystem at 1731025421407Initializing all the Stores at 1731025421410 (+3 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731025421410Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731025421412 (+2 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731025421412Cleaning up temporary data from old regions at 1731025421436 (+24 ms)Running coprocessor post-open hooks at 1731025421447 (+11 ms)Region opened successfully at 1731025421447 2024-11-08T00:23:41,450 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., pid=6, masterSystemTime=1731025421395 2024-11-08T00:23:41,456 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:41,457 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=b211e3aadde87eeeb10e4f0dad149691, regionState=OPEN, openSeqNum=2, regionLocation=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:41,456 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:41,463 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 because future has completed 2024-11-08T00:23:41,477 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-08T00:23:41,477 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 in 231 msec 2024-11-08T00:23:41,486 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-08T00:23:41,486 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b211e3aadde87eeeb10e4f0dad149691, ASSIGN in 406 msec 2024-11-08T00:23:41,488 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T00:23:41,489 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731025421489"}]},"ts":"1731025421489"} 2024-11-08T00:23:41,493 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-08T00:23:41,496 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T00:23:41,502 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 524 msec 2024-11-08T00:23:41,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T00:23:41,639 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:23:41,644 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fc411c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:23:41,644 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:23:41,644 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:23:41,647 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:23:41,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:23:41,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:23:41,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@eb5e284, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:23:41,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:23:41,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:23:41,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:23:41,651 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56464, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:23:41,653 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17282950, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:23:41,654 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:23:41,656 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:23:41,657 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:23:41,661 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53730, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:23:41,665 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-08T00:23:41,668 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 3302f0f507bd,36785,1731025414356 2024-11-08T00:23:41,668 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@24720917 2024-11-08T00:23:41,670 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-08T00:23:41,679 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56478, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-08T00:23:41,689 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a31de58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:23:41,689 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:23:41,689 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:23:41,691 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:23:41,694 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:23:41,694 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:23:41,695 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ebcbe76, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:23:41,695 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:23:41,695 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:23:41,699 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:23:41,708 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56504, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:23:41,711 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1612488f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:23:41,717 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f3f8889, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:23:41,717 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:23:41,717 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:23:41,719 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:23:41,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:23:41,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:23:41,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43b9dd8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:23:41,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:23:41,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:23:41,724 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:23:41,728 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56520, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:23:41,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ff1f1f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:23:41,735 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4631cba3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:23:41,736 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:23:41,736 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:23:41,745 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:23:41,746 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:23:41,746 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:23:41,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fc4f8ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:23:41,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:23:41,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:23:41,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:23:41,751 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56538, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:23:41,762 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@90ab23d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:23:41,763 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57af71a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:23:41,763 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:23:41,764 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:23:41,768 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:23:41,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:23:41,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:23:41,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d5c6971, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:23:41,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:23:41,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:23:41,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:23:41,776 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56564, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:23:41,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2287fe54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:23:41,783 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dfc79f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:23:41,783 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:23:41,784 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:23:41,788 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:23:41,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:23:41,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:23:41,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4440149b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:23:41,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:23:41,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:23:41,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:23:41,792 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56586, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:23:41,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34a4a7f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:23:41,803 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c8c49bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:23:41,803 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:23:41,804 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:23:41,805 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:23:41,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:23:41,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:23:41,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6887c109, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:23:41,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:23:41,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:23:41,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:23:41,811 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56594, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:23:41,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@156b1d0a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:23:41,822 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@309f8170, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:23:41,822 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:23:41,822 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:23:41,824 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:23:41,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:23:41,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:23:41,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48230300, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:23:41,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:23:41,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:23:41,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:23:41,827 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56610, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:23:41,829 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10ba8772, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:23:41,833 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c04eae4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:23:41,833 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:23:41,834 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:23:41,837 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:23:41,838 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:23:41,839 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:23:41,839 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@272ed7e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:23:41,839 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:23:41,840 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:23:41,840 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:23:41,842 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56624, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:23:41,844 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@477e84a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:23:41,847 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@572a18fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:23:41,847 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:23:41,847 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:23:41,849 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:23:41,850 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:23:41,850 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:23:41,851 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@233ee7fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:23:41,851 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:23:41,851 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:23:41,852 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:23:41,855 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56646, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:23:41,857 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@439eab3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:23:41,864 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:23:41,870 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:23:41,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:23:41,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:23:41,873 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-08T00:23:41,874 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:23:41,875 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47466, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:23:41,876 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:23:41,876 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:23:41,877 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:23:41,879 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:23:41,879 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47468, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:23:41,879 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:23:41,879 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:23:41,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-08T00:23:41,883 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47478, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:23:41,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-08T00:23:41,884 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:23:41,884 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:23:41,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:23:41,886 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T00:23:41,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:23:41,886 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:23:41,888 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:23:41,889 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47486, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:23:41,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2] 2024-11-08T00:23:41,889 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2] 2024-11-08T00:23:41,889 DEBUG [Thread-145 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:23:41,890 DEBUG [Thread-145 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:23:41,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2] 2024-11-08T00:23:41,890 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T00:23:41,890 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47480, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:23:41,890 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:23:41,891 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:23:41,891 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47492, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:23:41,891 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T00:23:41,893 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:23:41,895 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47506, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:23:41,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2] 2024-11-08T00:23:41,895 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2] 2024-11-08T00:23:41,897 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:23:41,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:23:41,902 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:23:41,904 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47520, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:23:41,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2] 2024-11-08T00:23:41,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2] 2024-11-08T00:23:41,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:23:41,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2] 2024-11-08T00:23:41,920 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:23:41,922 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47524, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:23:41,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2] 2024-11-08T00:23:41,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-08T00:23:42,010 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:23:42,015 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b211e3aadde87eeeb10e4f0dad149691 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T00:23:42,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on b211e3aadde87eeeb10e4f0dad149691 2024-11-08T00:23:42,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-08T00:23:42,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:42,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. as already flushing 2024-11-08T00:23:42,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:42,093 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:42,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:42,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:42,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-08T00:23:42,264 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/91bbc5fe702443389e0f9734f8611b87 is 50, key is test_row_0/A:col10/1731025421940/Put/seqid=0 2024-11-08T00:23:42,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-08T00:23:42,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:42,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47492 deadline: 1731025482288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:42,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:42,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:42,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47524 deadline: 1731025482295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:42,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:42,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:42,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:42,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:42,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:42,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:42,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47486 deadline: 1731025482301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:42,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:42,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:42,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:42,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:42,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47466 deadline: 1731025482307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:42,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:42,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:42,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:42,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:42,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47520 deadline: 1731025482312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:42,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:42,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:42,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:42,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:42,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. as already flushing 2024-11-08T00:23:42,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:42,358 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:42,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:42,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:42,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741837_1013 (size=14341) 2024-11-08T00:23:42,431 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/91bbc5fe702443389e0f9734f8611b87 2024-11-08T00:23:42,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:42,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47492 deadline: 1731025482439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:42,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:42,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:42,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:42,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:42,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:42,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47466 deadline: 1731025482440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:42,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47486 deadline: 1731025482440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:42,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:42,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:42,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:42,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:42,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47524 deadline: 1731025482440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:42,452 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:42,452 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:42,452 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:42,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:42,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:42,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:42,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:42,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47520 deadline: 1731025482448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:42,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:42,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:42,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:42,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-08T00:23:42,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-08T00:23:42,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:42,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. as already flushing 2024-11-08T00:23:42,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:42,534 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:42,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:42,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:42,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:42,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47466 deadline: 1731025482660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:42,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:42,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:42,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:42,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:42,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47486 deadline: 1731025482661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:42,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:42,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:42,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47492 deadline: 1731025482661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:42,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:42,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:42,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:42,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:42,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:42,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:42,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47524 deadline: 1731025482661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:42,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:42,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:42,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:42,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:42,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47520 deadline: 1731025482668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:42,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:42,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:42,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:42,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-08T00:23:42,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:42,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. as already flushing 2024-11-08T00:23:42,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:42,697 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:42,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:42,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:42,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/e9c38e2ce4fb48de8f8f3ca1d37be541 is 50, key is test_row_0/B:col10/1731025421940/Put/seqid=0 2024-11-08T00:23:42,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741838_1014 (size=12001) 2024-11-08T00:23:42,780 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/e9c38e2ce4fb48de8f8f3ca1d37be541 2024-11-08T00:23:42,862 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-08T00:23:42,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:42,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. as already flushing 2024-11-08T00:23:42,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:42,863 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:42,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:42,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:42,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/a76bf377a91f412f9b0bd2162c3efb82 is 50, key is test_row_0/C:col10/1731025421940/Put/seqid=0 2024-11-08T00:23:42,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741839_1015 (size=12001) 2024-11-08T00:23:42,958 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/a76bf377a91f412f9b0bd2162c3efb82 2024-11-08T00:23:42,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:42,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47486 deadline: 1731025482982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:42,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:42,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47466 deadline: 1731025482983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:42,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:42,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:42,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:42,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:42,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:42,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:42,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:42,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47492 deadline: 1731025482984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:42,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:42,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:42,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:42,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:42,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47524 deadline: 1731025482984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:42,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:42,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:42,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:43,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:43,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/91bbc5fe702443389e0f9734f8611b87 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/91bbc5fe702443389e0f9734f8611b87 2024-11-08T00:23:43,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-08T00:23:43,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47520 deadline: 1731025482994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:43,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:43,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:43,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:43,028 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-08T00:23:43,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:43,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. as already flushing 2024-11-08T00:23:43,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:43,029 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:43,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:43,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:43,037 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/91bbc5fe702443389e0f9734f8611b87, entries=200, sequenceid=13, filesize=14.0 K 2024-11-08T00:23:43,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/e9c38e2ce4fb48de8f8f3ca1d37be541 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/e9c38e2ce4fb48de8f8f3ca1d37be541 2024-11-08T00:23:43,074 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/e9c38e2ce4fb48de8f8f3ca1d37be541, entries=150, sequenceid=13, filesize=11.7 K 2024-11-08T00:23:43,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/a76bf377a91f412f9b0bd2162c3efb82 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/a76bf377a91f412f9b0bd2162c3efb82 2024-11-08T00:23:43,109 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/a76bf377a91f412f9b0bd2162c3efb82, entries=150, sequenceid=13, filesize=11.7 K 2024-11-08T00:23:43,118 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for b211e3aadde87eeeb10e4f0dad149691 in 1101ms, sequenceid=13, compaction requested=false 2024-11-08T00:23:43,122 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-08T00:23:43,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b211e3aadde87eeeb10e4f0dad149691: 2024-11-08T00:23:43,194 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-08T00:23:43,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:43,204 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing b211e3aadde87eeeb10e4f0dad149691 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-08T00:23:43,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/d22f82be612249ccb30f622750dec257 is 50, key is test_row_0/A:col10/1731025422296/Put/seqid=0 2024-11-08T00:23:43,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741840_1016 (size=12001) 2024-11-08T00:23:43,301 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/d22f82be612249ccb30f622750dec257 2024-11-08T00:23:43,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/cff1bf3a5e50474d94bec51db2b9b811 is 50, key is test_row_0/B:col10/1731025422296/Put/seqid=0 2024-11-08T00:23:43,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741841_1017 (size=12001) 2024-11-08T00:23:43,400 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/cff1bf3a5e50474d94bec51db2b9b811 2024-11-08T00:23:43,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/cf2cbcadc64f4611b569592abf87a531 is 50, key is test_row_0/C:col10/1731025422296/Put/seqid=0 2024-11-08T00:23:43,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741842_1018 (size=12001) 2024-11-08T00:23:43,482 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/cf2cbcadc64f4611b569592abf87a531 2024-11-08T00:23:43,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/d22f82be612249ccb30f622750dec257 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/d22f82be612249ccb30f622750dec257 2024-11-08T00:23:43,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on b211e3aadde87eeeb10e4f0dad149691 2024-11-08T00:23:43,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. as already flushing 2024-11-08T00:23:43,530 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/d22f82be612249ccb30f622750dec257, entries=150, sequenceid=38, filesize=11.7 K 2024-11-08T00:23:43,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/cff1bf3a5e50474d94bec51db2b9b811 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/cff1bf3a5e50474d94bec51db2b9b811 2024-11-08T00:23:43,557 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/cff1bf3a5e50474d94bec51db2b9b811, entries=150, sequenceid=38, filesize=11.7 K 2024-11-08T00:23:43,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/cf2cbcadc64f4611b569592abf87a531 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/cf2cbcadc64f4611b569592abf87a531 2024-11-08T00:23:43,585 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/cf2cbcadc64f4611b569592abf87a531, entries=150, sequenceid=38, filesize=11.7 K 2024-11-08T00:23:43,589 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=46.96 KB/48090 for b211e3aadde87eeeb10e4f0dad149691 in 385ms, sequenceid=38, compaction requested=false 2024-11-08T00:23:43,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for b211e3aadde87eeeb10e4f0dad149691: 2024-11-08T00:23:43,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:43,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-08T00:23:43,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-08T00:23:43,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on b211e3aadde87eeeb10e4f0dad149691 2024-11-08T00:23:43,592 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:23:43,592 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b211e3aadde87eeeb10e4f0dad149691 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T00:23:43,601 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-08T00:23:43,602 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7040 sec 2024-11-08T00:23:43,610 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/123a0cbb25724390bcbfe282f65a1354 is 50, key is test_row_0/A:col10/1731025423584/Put/seqid=0 2024-11-08T00:23:43,620 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.7290 sec 2024-11-08T00:23:43,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741843_1019 (size=19017) 2024-11-08T00:23:43,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:43,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47466 deadline: 1731025483747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:43,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:43,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:43,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:43,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:43,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47524 deadline: 1731025483775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:43,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:43,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:43,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:43,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:43,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47486 deadline: 1731025483788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:43,829 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:43,829 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:43,829 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:43,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:43,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47520 deadline: 1731025483794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:43,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:43,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:43,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:43,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:43,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47492 deadline: 1731025483819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:43,852 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:43,852 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:43,852 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:43,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:43,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47466 deadline: 1731025483908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:43,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:43,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:43,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:43,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:43,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47524 deadline: 1731025483908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:43,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:43,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:43,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:43,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:43,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47486 deadline: 1731025483943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:43,946 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:43,946 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:43,946 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:43,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:43,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47520 deadline: 1731025483943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:43,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:43,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:43,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:43,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:43,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47492 deadline: 1731025483973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:43,982 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:43,982 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:43,982 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:44,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-08T00:23:44,029 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:23:44,032 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-08T00:23:44,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-08T00:23:44,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-08T00:23:44,037 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T00:23:44,039 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T00:23:44,039 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T00:23:44,078 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/123a0cbb25724390bcbfe282f65a1354 2024-11-08T00:23:44,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:44,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47466 deadline: 1731025484118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:44,137 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:44,137 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:44,137 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:44,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:44,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47524 deadline: 1731025484118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:44,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/4844bc66e8924cacaff69f7993ddda06 is 50, key is test_row_0/B:col10/1731025423584/Put/seqid=0 2024-11-08T00:23:44,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:44,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:44,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:44,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-08T00:23:44,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:44,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47486 deadline: 1731025484158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:44,170 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:44,170 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:44,170 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:44,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:44,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47520 deadline: 1731025484158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:44,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:44,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:44,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:44,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741844_1020 (size=9657) 2024-11-08T00:23:44,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-08T00:23:44,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:44,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. as already flushing 2024-11-08T00:23:44,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:44,196 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:44,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:44,198 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/4844bc66e8924cacaff69f7993ddda06 2024-11-08T00:23:44,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:44,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47492 deadline: 1731025484189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:44,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:44,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:44,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:44,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:44,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/29dd74f6fc1b42d7a7a1a07ff1056dfd is 50, key is test_row_0/C:col10/1731025423584/Put/seqid=0 2024-11-08T00:23:44,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741845_1021 (size=9657) 2024-11-08T00:23:44,276 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/29dd74f6fc1b42d7a7a1a07ff1056dfd 2024-11-08T00:23:44,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/123a0cbb25724390bcbfe282f65a1354 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/123a0cbb25724390bcbfe282f65a1354 2024-11-08T00:23:44,304 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/123a0cbb25724390bcbfe282f65a1354, entries=300, sequenceid=50, filesize=18.6 K 2024-11-08T00:23:44,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/4844bc66e8924cacaff69f7993ddda06 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/4844bc66e8924cacaff69f7993ddda06 2024-11-08T00:23:44,324 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/4844bc66e8924cacaff69f7993ddda06, entries=100, sequenceid=50, filesize=9.4 K 2024-11-08T00:23:44,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/29dd74f6fc1b42d7a7a1a07ff1056dfd as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/29dd74f6fc1b42d7a7a1a07ff1056dfd 2024-11-08T00:23:44,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-08T00:23:44,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:44,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. as already flushing 2024-11-08T00:23:44,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:44,355 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:44,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:44,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:44,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-08T00:23:44,360 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/29dd74f6fc1b42d7a7a1a07ff1056dfd, entries=100, sequenceid=50, filesize=9.4 K 2024-11-08T00:23:44,362 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for b211e3aadde87eeeb10e4f0dad149691 in 770ms, sequenceid=50, compaction requested=true 2024-11-08T00:23:44,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b211e3aadde87eeeb10e4f0dad149691: 2024-11-08T00:23:44,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b211e3aadde87eeeb10e4f0dad149691:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T00:23:44,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:23:44,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b211e3aadde87eeeb10e4f0dad149691:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T00:23:44,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:23:44,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b211e3aadde87eeeb10e4f0dad149691:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:23:44,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:23:44,370 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:23:44,374 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:23:44,374 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:23:44,376 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): b211e3aadde87eeeb10e4f0dad149691/B is initiating minor compaction (all files) 2024-11-08T00:23:44,377 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b211e3aadde87eeeb10e4f0dad149691/B in TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:44,377 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/e9c38e2ce4fb48de8f8f3ca1d37be541, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/cff1bf3a5e50474d94bec51db2b9b811, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/4844bc66e8924cacaff69f7993ddda06] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp, totalSize=32.9 K 2024-11-08T00:23:44,382 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting e9c38e2ce4fb48de8f8f3ca1d37be541, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731025421940 2024-11-08T00:23:44,385 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting cff1bf3a5e50474d94bec51db2b9b811, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731025422277 2024-11-08T00:23:44,386 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 4844bc66e8924cacaff69f7993ddda06, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731025423540 2024-11-08T00:23:44,389 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 45359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:23:44,390 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): b211e3aadde87eeeb10e4f0dad149691/A is initiating minor compaction (all files) 2024-11-08T00:23:44,390 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b211e3aadde87eeeb10e4f0dad149691/A in TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:44,390 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/91bbc5fe702443389e0f9734f8611b87, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/d22f82be612249ccb30f622750dec257, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/123a0cbb25724390bcbfe282f65a1354] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp, totalSize=44.3 K 2024-11-08T00:23:44,393 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91bbc5fe702443389e0f9734f8611b87, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731025421940 2024-11-08T00:23:44,394 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting d22f82be612249ccb30f622750dec257, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731025422277 2024-11-08T00:23:44,398 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 123a0cbb25724390bcbfe282f65a1354, keycount=300, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731025423516 2024-11-08T00:23:44,452 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b211e3aadde87eeeb10e4f0dad149691#B#compaction#9 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:23:44,454 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/960ca2b1bc6847eeba29b013dfa5d384 is 50, key is test_row_0/B:col10/1731025423584/Put/seqid=0 2024-11-08T00:23:44,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on b211e3aadde87eeeb10e4f0dad149691 2024-11-08T00:23:44,460 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:23:44,461 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b211e3aadde87eeeb10e4f0dad149691 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-08T00:23:44,479 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b211e3aadde87eeeb10e4f0dad149691#A#compaction#10 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:23:44,482 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/d14b3c474e8b4937ab4416e97f37d803 is 50, key is test_row_0/A:col10/1731025423584/Put/seqid=0 2024-11-08T00:23:44,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-08T00:23:44,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:44,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. as already flushing 2024-11-08T00:23:44,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:44,512 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:44,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:44,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:44,524 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/d4985818e7d648d68004e55a13d5dfbb is 50, key is test_row_0/A:col10/1731025423727/Put/seqid=0 2024-11-08T00:23:44,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:44,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47486 deadline: 1731025484500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:44,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:44,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:44,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:44,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741846_1022 (size=12104) 2024-11-08T00:23:44,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:44,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47492 deadline: 1731025484510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:44,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:44,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:44,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:44,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:44,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47524 deadline: 1731025484525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:44,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:44,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:44,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:44,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:44,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47466 deadline: 1731025484530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:44,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:44,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47520 deadline: 1731025484533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:44,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:44,557 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:44,557 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:44,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:44,560 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/960ca2b1bc6847eeba29b013dfa5d384 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/960ca2b1bc6847eeba29b013dfa5d384 2024-11-08T00:23:44,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:44,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:44,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741847_1023 (size=12104) 2024-11-08T00:23:44,600 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/d14b3c474e8b4937ab4416e97f37d803 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/d14b3c474e8b4937ab4416e97f37d803 2024-11-08T00:23:44,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741848_1024 (size=16681) 2024-11-08T00:23:44,644 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b211e3aadde87eeeb10e4f0dad149691/B of b211e3aadde87eeeb10e4f0dad149691 into 960ca2b1bc6847eeba29b013dfa5d384(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:23:44,650 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b211e3aadde87eeeb10e4f0dad149691/A of b211e3aadde87eeeb10e4f0dad149691 into d14b3c474e8b4937ab4416e97f37d803(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:23:44,651 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b211e3aadde87eeeb10e4f0dad149691: 2024-11-08T00:23:44,651 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b211e3aadde87eeeb10e4f0dad149691: 2024-11-08T00:23:44,656 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., storeName=b211e3aadde87eeeb10e4f0dad149691/A, priority=13, startTime=1731025424364; duration=0sec 2024-11-08T00:23:44,659 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:23:44,659 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b211e3aadde87eeeb10e4f0dad149691:A 2024-11-08T00:23:44,660 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:23:44,663 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:23:44,663 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): b211e3aadde87eeeb10e4f0dad149691/C is initiating minor compaction (all files) 2024-11-08T00:23:44,663 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b211e3aadde87eeeb10e4f0dad149691/C in TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:44,664 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/a76bf377a91f412f9b0bd2162c3efb82, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/cf2cbcadc64f4611b569592abf87a531, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/29dd74f6fc1b42d7a7a1a07ff1056dfd] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp, totalSize=32.9 K 2024-11-08T00:23:44,664 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., storeName=b211e3aadde87eeeb10e4f0dad149691/B, priority=13, startTime=1731025424367; duration=0sec 2024-11-08T00:23:44,664 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:23:44,664 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b211e3aadde87eeeb10e4f0dad149691:B 2024-11-08T00:23:44,665 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting a76bf377a91f412f9b0bd2162c3efb82, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731025421940 2024-11-08T00:23:44,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-08T00:23:44,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-08T00:23:44,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:44,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. as already flushing 2024-11-08T00:23:44,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:44,674 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:44,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:44,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:44,677 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf2cbcadc64f4611b569592abf87a531, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731025422277 2024-11-08T00:23:44,679 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29dd74f6fc1b42d7a7a1a07ff1056dfd, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731025423540 2024-11-08T00:23:44,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:44,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47486 deadline: 1731025484641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:44,684 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:44,684 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:44,684 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:44,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:44,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47524 deadline: 1731025484658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:44,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:44,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:44,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:44,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:44,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47520 deadline: 1731025484668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:44,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:44,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47466 deadline: 1731025484692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:44,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:44,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:44,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:44,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:44,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:44,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:44,737 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b211e3aadde87eeeb10e4f0dad149691#C#compaction#12 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:23:44,739 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/9ed79859e52e4defb670125cce956c03 is 50, key is test_row_0/C:col10/1731025423584/Put/seqid=0 2024-11-08T00:23:44,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741849_1025 (size=12104) 2024-11-08T00:23:44,799 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/9ed79859e52e4defb670125cce956c03 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/9ed79859e52e4defb670125cce956c03 2024-11-08T00:23:44,822 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b211e3aadde87eeeb10e4f0dad149691/C of b211e3aadde87eeeb10e4f0dad149691 into 9ed79859e52e4defb670125cce956c03(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:23:44,823 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b211e3aadde87eeeb10e4f0dad149691: 2024-11-08T00:23:44,823 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., storeName=b211e3aadde87eeeb10e4f0dad149691/C, priority=13, startTime=1731025424368; duration=0sec 2024-11-08T00:23:44,823 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:23:44,823 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b211e3aadde87eeeb10e4f0dad149691:C 2024-11-08T00:23:44,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-08T00:23:44,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:44,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. as already flushing 2024-11-08T00:23:44,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:44,845 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:44,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:44,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:44,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:44,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47486 deadline: 1731025484887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:44,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:44,898 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:44,898 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:44,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:44,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47524 deadline: 1731025484898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:44,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:44,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:44,901 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:44,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:44,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47520 deadline: 1731025484908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:44,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:44,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:44,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:44,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:44,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47466 deadline: 1731025484912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:44,917 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:44,917 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:44,917 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:45,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-08T00:23:45,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:45,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. as already flushing 2024-11-08T00:23:45,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:45,001 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:45,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:45,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:45,032 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/d4985818e7d648d68004e55a13d5dfbb 2024-11-08T00:23:45,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:45,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47492 deadline: 1731025485040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:45,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:45,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:45,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:45,056 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-08T00:23:45,056 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-08T00:23:45,079 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/2cd4bfaeb8294572a28fc37728698047 is 50, key is test_row_0/B:col10/1731025423727/Put/seqid=0 2024-11-08T00:23:45,093 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-08T00:23:45,094 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-08T00:23:45,094 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-08T00:23:45,094 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-08T00:23:45,113 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-08T00:23:45,113 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-08T00:23:45,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741850_1026 (size=12001) 2024-11-08T00:23:45,126 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/2cd4bfaeb8294572a28fc37728698047 2024-11-08T00:23:45,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-08T00:23:45,157 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/2cd8e63598b04501ad52d541451ee148 is 50, key is test_row_0/C:col10/1731025423727/Put/seqid=0 2024-11-08T00:23:45,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:45,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. as already flushing 2024-11-08T00:23:45,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:45,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:45,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:45,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:45,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741851_1027 (size=12001) 2024-11-08T00:23:45,174 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/2cd8e63598b04501ad52d541451ee148 2024-11-08T00:23:45,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-08T00:23:45,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/d4985818e7d648d68004e55a13d5dfbb as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/d4985818e7d648d68004e55a13d5dfbb 2024-11-08T00:23:45,205 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/d4985818e7d648d68004e55a13d5dfbb, entries=250, sequenceid=77, filesize=16.3 K 2024-11-08T00:23:45,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:45,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47524 deadline: 1731025485208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:45,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:45,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:45,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:45,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:45,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47486 deadline: 1731025485209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:45,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/2cd4bfaeb8294572a28fc37728698047 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/2cd4bfaeb8294572a28fc37728698047 2024-11-08T00:23:45,219 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:45,219 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:45,219 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:45,233 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/2cd4bfaeb8294572a28fc37728698047, entries=150, sequenceid=77, filesize=11.7 K 2024-11-08T00:23:45,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/2cd8e63598b04501ad52d541451ee148 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/2cd8e63598b04501ad52d541451ee148 2024-11-08T00:23:45,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:45,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47520 deadline: 1731025485228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:45,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:45,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47466 deadline: 1731025485228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:45,249 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:45,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:45,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:45,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:45,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:45,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:45,255 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/2cd8e63598b04501ad52d541451ee148, entries=150, sequenceid=77, filesize=11.7 K 2024-11-08T00:23:45,262 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for b211e3aadde87eeeb10e4f0dad149691 in 801ms, sequenceid=77, compaction requested=false 2024-11-08T00:23:45,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b211e3aadde87eeeb10e4f0dad149691: 2024-11-08T00:23:45,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-08T00:23:45,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:45,316 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing b211e3aadde87eeeb10e4f0dad149691 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-08T00:23:45,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/a7368cc2155944a39cc48aabb0fd0c60 is 50, key is test_row_0/A:col10/1731025424510/Put/seqid=0 2024-11-08T00:23:45,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741852_1028 (size=12001) 2024-11-08T00:23:45,391 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/a7368cc2155944a39cc48aabb0fd0c60 2024-11-08T00:23:45,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/ab5437c759354644904545b32e364872 is 50, key is test_row_0/B:col10/1731025424510/Put/seqid=0 2024-11-08T00:23:45,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741853_1029 (size=12001) 2024-11-08T00:23:45,470 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/ab5437c759354644904545b32e364872 2024-11-08T00:23:45,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/5ded9cb9e5d64f62ab7e0843cfb217b6 is 50, key is test_row_0/C:col10/1731025424510/Put/seqid=0 2024-11-08T00:23:45,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741854_1030 (size=12001) 2024-11-08T00:23:45,563 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/5ded9cb9e5d64f62ab7e0843cfb217b6 2024-11-08T00:23:45,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/a7368cc2155944a39cc48aabb0fd0c60 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/a7368cc2155944a39cc48aabb0fd0c60 2024-11-08T00:23:45,604 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/a7368cc2155944a39cc48aabb0fd0c60, entries=150, sequenceid=90, filesize=11.7 K 2024-11-08T00:23:45,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/ab5437c759354644904545b32e364872 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/ab5437c759354644904545b32e364872 2024-11-08T00:23:45,639 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/ab5437c759354644904545b32e364872, entries=150, sequenceid=90, filesize=11.7 K 2024-11-08T00:23:45,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/5ded9cb9e5d64f62ab7e0843cfb217b6 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/5ded9cb9e5d64f62ab7e0843cfb217b6 2024-11-08T00:23:45,662 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/5ded9cb9e5d64f62ab7e0843cfb217b6, entries=150, sequenceid=90, filesize=11.7 K 2024-11-08T00:23:45,668 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=0 B/0 for b211e3aadde87eeeb10e4f0dad149691 in 351ms, sequenceid=90, compaction requested=true 2024-11-08T00:23:45,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for b211e3aadde87eeeb10e4f0dad149691: 2024-11-08T00:23:45,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:45,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-08T00:23:45,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-08T00:23:45,679 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-08T00:23:45,679 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6340 sec 2024-11-08T00:23:45,698 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.6500 sec 2024-11-08T00:23:45,786 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:23:45,786 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b211e3aadde87eeeb10e4f0dad149691 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T00:23:45,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on b211e3aadde87eeeb10e4f0dad149691 2024-11-08T00:23:45,804 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/fdd0a475c7f44b5a88b4c51fdde08618 is 50, key is test_row_0/A:col10/1731025425785/Put/seqid=0 2024-11-08T00:23:45,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741855_1031 (size=21365) 2024-11-08T00:23:45,835 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/fdd0a475c7f44b5a88b4c51fdde08618 2024-11-08T00:23:45,854 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T00:23:45,884 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/59a5b8b6c0a54bb2812effa120328eb1 is 50, key is test_row_0/B:col10/1731025425785/Put/seqid=0 2024-11-08T00:23:45,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741856_1032 (size=12001) 2024-11-08T00:23:45,935 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/59a5b8b6c0a54bb2812effa120328eb1 2024-11-08T00:23:45,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:45,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47524 deadline: 1731025485912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:45,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:45,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:45,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:45,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:45,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47466 deadline: 1731025485916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:45,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:45,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:45,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:45,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:45,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47486 deadline: 1731025485919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:45,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:45,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:45,947 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:45,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:45,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47520 deadline: 1731025485935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:45,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:45,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:45,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:45,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/433ba9aae46847c3a2c3e661f61d8660 is 50, key is test_row_0/C:col10/1731025425785/Put/seqid=0 2024-11-08T00:23:45,995 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-08T00:23:46,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741857_1033 (size=12001) 2024-11-08T00:23:46,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:46,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:46,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47466 deadline: 1731025486048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:46,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:46,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:46,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:46,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47524 deadline: 1731025486048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:46,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:46,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:46,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:46,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:46,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47486 deadline: 1731025486059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:46,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:46,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:46,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:46,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:46,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47492 deadline: 1731025486067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:46,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:46,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47520 deadline: 1731025486078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:46,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:46,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:46,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:46,086 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 3302f0f507bd,40479,1731025415856 for 'test_row_1' in b211e3aadde87eeeb10e4f0dad149691 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2255 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T00:23:46,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:46,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:46,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:46,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-08T00:23:46,199 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:23:46,200 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-08T00:23:46,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-08T00:23:46,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-08T00:23:46,213 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T00:23:46,219 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T00:23:46,219 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T00:23:46,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:46,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47466 deadline: 1731025486268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:46,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:46,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:46,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:46,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:46,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47486 deadline: 1731025486278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:46,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:46,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47524 deadline: 1731025486278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:46,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:46,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:46,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:46,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:46,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:46,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:46,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:46,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47520 deadline: 1731025486298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:46,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:46,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:46,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:46,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-08T00:23:46,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-08T00:23:46,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:46,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. as already flushing 2024-11-08T00:23:46,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:46,390 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:46,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:46,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:46,446 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/433ba9aae46847c3a2c3e661f61d8660 2024-11-08T00:23:46,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/fdd0a475c7f44b5a88b4c51fdde08618 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/fdd0a475c7f44b5a88b4c51fdde08618 2024-11-08T00:23:46,489 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/fdd0a475c7f44b5a88b4c51fdde08618, entries=350, sequenceid=102, filesize=20.9 K 2024-11-08T00:23:46,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/59a5b8b6c0a54bb2812effa120328eb1 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/59a5b8b6c0a54bb2812effa120328eb1 2024-11-08T00:23:46,528 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/59a5b8b6c0a54bb2812effa120328eb1, entries=150, sequenceid=102, filesize=11.7 K 2024-11-08T00:23:46,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/433ba9aae46847c3a2c3e661f61d8660 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/433ba9aae46847c3a2c3e661f61d8660 2024-11-08T00:23:46,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-08T00:23:46,547 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-08T00:23:46,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:46,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. as already flushing 2024-11-08T00:23:46,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:46,548 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:46,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:46,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:46,553 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/433ba9aae46847c3a2c3e661f61d8660, entries=150, sequenceid=102, filesize=11.7 K 2024-11-08T00:23:46,564 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for b211e3aadde87eeeb10e4f0dad149691 in 778ms, sequenceid=102, compaction requested=true 2024-11-08T00:23:46,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b211e3aadde87eeeb10e4f0dad149691: 2024-11-08T00:23:46,565 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T00:23:46,569 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 62151 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T00:23:46,569 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): b211e3aadde87eeeb10e4f0dad149691/A is initiating minor compaction (all files) 2024-11-08T00:23:46,570 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b211e3aadde87eeeb10e4f0dad149691/A in TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:46,570 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/d14b3c474e8b4937ab4416e97f37d803, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/d4985818e7d648d68004e55a13d5dfbb, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/a7368cc2155944a39cc48aabb0fd0c60, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/fdd0a475c7f44b5a88b4c51fdde08618] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp, totalSize=60.7 K 2024-11-08T00:23:46,572 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting d14b3c474e8b4937ab4416e97f37d803, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731025422277 2024-11-08T00:23:46,573 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4985818e7d648d68004e55a13d5dfbb, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731025423727 2024-11-08T00:23:46,574 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7368cc2155944a39cc48aabb0fd0c60, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731025424510 2024-11-08T00:23:46,575 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting fdd0a475c7f44b5a88b4c51fdde08618, keycount=350, bloomtype=ROW, size=20.9 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731025425742 2024-11-08T00:23:46,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b211e3aadde87eeeb10e4f0dad149691:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T00:23:46,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:23:46,589 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T00:23:46,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b211e3aadde87eeeb10e4f0dad149691:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T00:23:46,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:23:46,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b211e3aadde87eeeb10e4f0dad149691:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:23:46,593 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T00:23:46,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:23:46,593 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): b211e3aadde87eeeb10e4f0dad149691/B is initiating minor compaction (all files) 2024-11-08T00:23:46,593 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b211e3aadde87eeeb10e4f0dad149691/B in TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:46,594 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/960ca2b1bc6847eeba29b013dfa5d384, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/2cd4bfaeb8294572a28fc37728698047, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/ab5437c759354644904545b32e364872, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/59a5b8b6c0a54bb2812effa120328eb1] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp, totalSize=47.0 K 2024-11-08T00:23:46,595 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 960ca2b1bc6847eeba29b013dfa5d384, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731025422277 2024-11-08T00:23:46,596 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cd4bfaeb8294572a28fc37728698047, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731025423727 2024-11-08T00:23:46,599 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting ab5437c759354644904545b32e364872, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731025424510 2024-11-08T00:23:46,601 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 59a5b8b6c0a54bb2812effa120328eb1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731025425760 2024-11-08T00:23:46,604 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:23:46,605 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b211e3aadde87eeeb10e4f0dad149691 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-08T00:23:46,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on b211e3aadde87eeeb10e4f0dad149691 2024-11-08T00:23:46,649 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b211e3aadde87eeeb10e4f0dad149691#A#compaction#21 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:23:46,650 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/441d335b82014cd19a4de813ccc11a24 is 50, key is test_row_0/A:col10/1731025425785/Put/seqid=0 2024-11-08T00:23:46,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:46,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47520 deadline: 1731025486654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:46,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:46,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:46,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47486 deadline: 1731025486655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:46,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47466 deadline: 1731025486658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:46,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:46,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:46,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:46,666 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:46,666 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:46,666 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:46,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:46,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:46,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47524 deadline: 1731025486664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:46,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:46,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:46,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:46,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:46,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:46,673 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/283a02a889d040e6bfe3c9950e72f3c1 is 50, key is test_row_0/A:col10/1731025425910/Put/seqid=0 2024-11-08T00:23:46,699 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b211e3aadde87eeeb10e4f0dad149691#B#compaction#23 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:23:46,701 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/e2ae0a5be75d4e6ba65e0e36ada2966b is 50, key is test_row_0/B:col10/1731025425785/Put/seqid=0 2024-11-08T00:23:46,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-08T00:23:46,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:46,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. as already flushing 2024-11-08T00:23:46,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:46,713 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:46,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:46,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:46,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741858_1034 (size=12241) 2024-11-08T00:23:46,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741859_1035 (size=14341) 2024-11-08T00:23:46,786 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/283a02a889d040e6bfe3c9950e72f3c1 2024-11-08T00:23:46,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:46,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47520 deadline: 1731025486779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:46,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:46,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:46,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:46,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:46,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47486 deadline: 1731025486779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:46,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:46,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:46,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:46,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:46,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47524 deadline: 1731025486780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:46,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:46,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:46,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47466 deadline: 1731025486780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:46,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:46,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:46,797 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:46,797 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:46,797 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:46,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741860_1036 (size=12241) 2024-11-08T00:23:46,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/f28c43f5ab8444f6a7b91f495359a5f1 is 50, key is test_row_0/B:col10/1731025425910/Put/seqid=0 2024-11-08T00:23:46,829 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/e2ae0a5be75d4e6ba65e0e36ada2966b as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/e2ae0a5be75d4e6ba65e0e36ada2966b 2024-11-08T00:23:46,846 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b211e3aadde87eeeb10e4f0dad149691/B of b211e3aadde87eeeb10e4f0dad149691 into e2ae0a5be75d4e6ba65e0e36ada2966b(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:23:46,846 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b211e3aadde87eeeb10e4f0dad149691: 2024-11-08T00:23:46,846 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., storeName=b211e3aadde87eeeb10e4f0dad149691/B, priority=12, startTime=1731025426589; duration=0sec 2024-11-08T00:23:46,847 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:23:46,847 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b211e3aadde87eeeb10e4f0dad149691:B 2024-11-08T00:23:46,847 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T00:23:46,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-08T00:23:46,850 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T00:23:46,850 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): b211e3aadde87eeeb10e4f0dad149691/C is initiating minor compaction (all files) 2024-11-08T00:23:46,851 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b211e3aadde87eeeb10e4f0dad149691/C in TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:46,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741861_1037 (size=12001) 2024-11-08T00:23:46,853 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/9ed79859e52e4defb670125cce956c03, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/2cd8e63598b04501ad52d541451ee148, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/5ded9cb9e5d64f62ab7e0843cfb217b6, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/433ba9aae46847c3a2c3e661f61d8660] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp, totalSize=47.0 K 2024-11-08T00:23:46,853 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/f28c43f5ab8444f6a7b91f495359a5f1 2024-11-08T00:23:46,855 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ed79859e52e4defb670125cce956c03, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731025422277 2024-11-08T00:23:46,858 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cd8e63598b04501ad52d541451ee148, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731025423727 2024-11-08T00:23:46,859 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ded9cb9e5d64f62ab7e0843cfb217b6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731025424510 2024-11-08T00:23:46,861 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 433ba9aae46847c3a2c3e661f61d8660, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731025425760 2024-11-08T00:23:46,876 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-08T00:23:46,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:46,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. as already flushing 2024-11-08T00:23:46,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:46,877 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:46,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:46,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:46,881 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/f5e4ace533af47d8969ae95395a87fa0 is 50, key is test_row_0/C:col10/1731025425910/Put/seqid=0 2024-11-08T00:23:46,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741862_1038 (size=12001) 2024-11-08T00:23:46,920 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/f5e4ace533af47d8969ae95395a87fa0 2024-11-08T00:23:46,935 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b211e3aadde87eeeb10e4f0dad149691#C#compaction#26 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:23:46,949 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/2fa338d8b64243efbf48b133750301fe is 50, key is test_row_0/C:col10/1731025425785/Put/seqid=0 2024-11-08T00:23:46,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/283a02a889d040e6bfe3c9950e72f3c1 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/283a02a889d040e6bfe3c9950e72f3c1 2024-11-08T00:23:46,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741863_1039 (size=12241) 2024-11-08T00:23:47,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:47,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47520 deadline: 1731025486998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:47,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:47,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:47,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:47,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:47,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47486 deadline: 1731025487008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:47,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:47,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47524 deadline: 1731025487008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:47,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:47,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:47,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:47,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:47,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47466 deadline: 1731025487009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:47,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:47,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:47,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:47,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:47,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:47,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:47,038 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/2fa338d8b64243efbf48b133750301fe as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/2fa338d8b64243efbf48b133750301fe 2024-11-08T00:23:47,042 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/283a02a889d040e6bfe3c9950e72f3c1, entries=200, sequenceid=127, filesize=14.0 K 2024-11-08T00:23:47,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-08T00:23:47,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:47,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. as already flushing 2024-11-08T00:23:47,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:47,046 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:47,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:47,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:47,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/f28c43f5ab8444f6a7b91f495359a5f1 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/f28c43f5ab8444f6a7b91f495359a5f1 2024-11-08T00:23:47,084 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b211e3aadde87eeeb10e4f0dad149691/C of b211e3aadde87eeeb10e4f0dad149691 into 2fa338d8b64243efbf48b133750301fe(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:23:47,084 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b211e3aadde87eeeb10e4f0dad149691: 2024-11-08T00:23:47,084 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., storeName=b211e3aadde87eeeb10e4f0dad149691/C, priority=12, startTime=1731025426593; duration=0sec 2024-11-08T00:23:47,085 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:23:47,085 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b211e3aadde87eeeb10e4f0dad149691:C 2024-11-08T00:23:47,089 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/f28c43f5ab8444f6a7b91f495359a5f1, entries=150, sequenceid=127, filesize=11.7 K 2024-11-08T00:23:47,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/f5e4ace533af47d8969ae95395a87fa0 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/f5e4ace533af47d8969ae95395a87fa0 2024-11-08T00:23:47,111 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/f5e4ace533af47d8969ae95395a87fa0, entries=150, sequenceid=127, filesize=11.7 K 2024-11-08T00:23:47,117 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for b211e3aadde87eeeb10e4f0dad149691 in 513ms, sequenceid=127, compaction requested=false 2024-11-08T00:23:47,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b211e3aadde87eeeb10e4f0dad149691: 2024-11-08T00:23:47,196 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/441d335b82014cd19a4de813ccc11a24 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/441d335b82014cd19a4de813ccc11a24 2024-11-08T00:23:47,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-08T00:23:47,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:47,222 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing b211e3aadde87eeeb10e4f0dad149691 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T00:23:47,239 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b211e3aadde87eeeb10e4f0dad149691/A of b211e3aadde87eeeb10e4f0dad149691 into 441d335b82014cd19a4de813ccc11a24(size=12.0 K), total size for store is 26.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:23:47,239 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b211e3aadde87eeeb10e4f0dad149691: 2024-11-08T00:23:47,240 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., storeName=b211e3aadde87eeeb10e4f0dad149691/A, priority=12, startTime=1731025426565; duration=0sec 2024-11-08T00:23:47,240 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:23:47,240 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b211e3aadde87eeeb10e4f0dad149691:A 2024-11-08T00:23:47,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/f1769abfb5f04776884d7c9fb07e3d62 is 50, key is test_row_0/A:col10/1731025426660/Put/seqid=0 2024-11-08T00:23:47,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741864_1040 (size=9757) 2024-11-08T00:23:47,318 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/f1769abfb5f04776884d7c9fb07e3d62 2024-11-08T00:23:47,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. as already flushing 2024-11-08T00:23:47,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on b211e3aadde87eeeb10e4f0dad149691 2024-11-08T00:23:47,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/3c5be6f01f4e49f69b05148f93e8799e is 50, key is test_row_0/B:col10/1731025426660/Put/seqid=0 2024-11-08T00:23:47,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-08T00:23:47,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741865_1041 (size=9757) 2024-11-08T00:23:47,410 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/3c5be6f01f4e49f69b05148f93e8799e 2024-11-08T00:23:47,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:47,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47524 deadline: 1731025487425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:47,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:47,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:47,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:47,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/a82d8980c0ca4e598b8d282a6d79a4c0 is 50, key is test_row_0/C:col10/1731025426660/Put/seqid=0 2024-11-08T00:23:47,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:47,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:47,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47520 deadline: 1731025487434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:47,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:47,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:47,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:47,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47466 deadline: 1731025487433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:47,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:47,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:47,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:47,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:47,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47486 deadline: 1731025487436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:47,460 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:47,460 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:47,460 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:47,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741866_1042 (size=9757) 2024-11-08T00:23:47,491 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/a82d8980c0ca4e598b8d282a6d79a4c0 2024-11-08T00:23:47,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/f1769abfb5f04776884d7c9fb07e3d62 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/f1769abfb5f04776884d7c9fb07e3d62 2024-11-08T00:23:47,521 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/f1769abfb5f04776884d7c9fb07e3d62, entries=100, sequenceid=140, filesize=9.5 K 2024-11-08T00:23:47,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/3c5be6f01f4e49f69b05148f93e8799e as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/3c5be6f01f4e49f69b05148f93e8799e 2024-11-08T00:23:47,536 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/3c5be6f01f4e49f69b05148f93e8799e, entries=100, sequenceid=140, filesize=9.5 K 2024-11-08T00:23:47,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/a82d8980c0ca4e598b8d282a6d79a4c0 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/a82d8980c0ca4e598b8d282a6d79a4c0 2024-11-08T00:23:47,555 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/a82d8980c0ca4e598b8d282a6d79a4c0, entries=100, sequenceid=140, filesize=9.5 K 2024-11-08T00:23:47,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:47,557 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for b211e3aadde87eeeb10e4f0dad149691 in 335ms, sequenceid=140, compaction requested=true 2024-11-08T00:23:47,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for b211e3aadde87eeeb10e4f0dad149691: 2024-11-08T00:23:47,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:47,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-08T00:23:47,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47524 deadline: 1731025487538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:47,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-08T00:23:47,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:47,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:47,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:47,565 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-08T00:23:47,565 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3420 sec 2024-11-08T00:23:47,570 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.3670 sec 2024-11-08T00:23:47,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on b211e3aadde87eeeb10e4f0dad149691 2024-11-08T00:23:47,581 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:23:47,581 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b211e3aadde87eeeb10e4f0dad149691 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-08T00:23:47,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/0b4348ac313144ba97de5e878c672caf is 50, key is test_row_0/A:col10/1731025427574/Put/seqid=0 2024-11-08T00:23:47,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:47,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47486 deadline: 1731025487623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:47,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:47,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47466 deadline: 1731025487626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:47,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:47,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:47,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:47,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:47,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:47,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:47,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:47,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47520 deadline: 1731025487627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:47,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:47,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:47,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:47,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741867_1043 (size=16931) 2024-11-08T00:23:47,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:47,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47486 deadline: 1731025487748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:47,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:47,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:47,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:47,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:47,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47466 deadline: 1731025487748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:47,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:47,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47520 deadline: 1731025487748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:47,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:47,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:47,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:47,764 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:47,764 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:47,765 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:47,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:47,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47524 deadline: 1731025487769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:47,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:47,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:47,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:47,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:47,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47486 deadline: 1731025487968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:47,972 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:47,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:47,972 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:47,972 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:47,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47520 deadline: 1731025487969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:47,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:47,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:47,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:47,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:47,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47466 deadline: 1731025487969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:47,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:47,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:47,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:48,063 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/0b4348ac313144ba97de5e878c672caf 2024-11-08T00:23:48,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:48,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47524 deadline: 1731025488079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:48,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:48,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:48,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:48,107 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/9d8ca6499d774958846c2129dce08760 is 50, key is test_row_0/B:col10/1731025427574/Put/seqid=0 2024-11-08T00:23:48,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:48,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47492 deadline: 1731025488109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:48,113 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 3302f0f507bd,40479,1731025415856 for 'test_row_1' in b211e3aadde87eeeb10e4f0dad149691 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4295 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T00:23:48,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:48,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:48,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:48,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741868_1044 (size=12151) 2024-11-08T00:23:48,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:48,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47486 deadline: 1731025488278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:48,285 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:48,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:48,285 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:48,285 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:48,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47520 deadline: 1731025488278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:48,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:48,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:48,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:48,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:48,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47466 deadline: 1731025488287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:48,292 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:48,292 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:48,293 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:48,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-08T00:23:48,380 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:23:48,381 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-08T00:23:48,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-08T00:23:48,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-08T00:23:48,385 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T00:23:48,388 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T00:23:48,388 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T00:23:48,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-08T00:23:48,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-08T00:23:48,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:48,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. as already flushing 2024-11-08T00:23:48,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:48,544 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:48,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:48,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:48,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:48,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47524 deadline: 1731025488607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:48,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:48,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:48,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:48,642 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/9d8ca6499d774958846c2129dce08760 2024-11-08T00:23:48,669 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/ac8b7963794349d395f49cda6ce4d286 is 50, key is test_row_0/C:col10/1731025427574/Put/seqid=0 2024-11-08T00:23:48,702 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-08T00:23:48,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:48,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. as already flushing 2024-11-08T00:23:48,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:48,703 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:48,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:48,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:48,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-08T00:23:48,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741869_1045 (size=12151) 2024-11-08T00:23:48,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:48,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47520 deadline: 1731025488797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:48,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:48,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:48,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:48,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:48,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47486 deadline: 1731025488798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:48,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:48,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:48,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:48,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:48,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47466 deadline: 1731025488798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:48,801 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:48,801 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:48,801 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:48,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-08T00:23:48,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:48,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. as already flushing 2024-11-08T00:23:48,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:48,858 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:48,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:48,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:49,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-08T00:23:49,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:49,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. as already flushing 2024-11-08T00:23:49,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:49,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-08T00:23:49,018 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:49,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:49,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:49,137 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/ac8b7963794349d395f49cda6ce4d286 2024-11-08T00:23:49,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/0b4348ac313144ba97de5e878c672caf as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/0b4348ac313144ba97de5e878c672caf 2024-11-08T00:23:49,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-08T00:23:49,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:49,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. as already flushing 2024-11-08T00:23:49,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:49,174 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:49,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:49,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:49,198 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/0b4348ac313144ba97de5e878c672caf, entries=250, sequenceid=169, filesize=16.5 K 2024-11-08T00:23:49,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/9d8ca6499d774958846c2129dce08760 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/9d8ca6499d774958846c2129dce08760 2024-11-08T00:23:49,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/9d8ca6499d774958846c2129dce08760, entries=150, sequenceid=169, filesize=11.9 K 2024-11-08T00:23:49,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/ac8b7963794349d395f49cda6ce4d286 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/ac8b7963794349d395f49cda6ce4d286 2024-11-08T00:23:49,252 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/ac8b7963794349d395f49cda6ce4d286, entries=150, sequenceid=169, filesize=11.9 K 2024-11-08T00:23:49,256 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for b211e3aadde87eeeb10e4f0dad149691 in 1673ms, sequenceid=169, compaction requested=true 2024-11-08T00:23:49,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b211e3aadde87eeeb10e4f0dad149691: 2024-11-08T00:23:49,256 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T00:23:49,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b211e3aadde87eeeb10e4f0dad149691:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T00:23:49,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:23:49,259 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53270 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T00:23:49,259 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): b211e3aadde87eeeb10e4f0dad149691/A is initiating minor compaction (all files) 2024-11-08T00:23:49,260 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T00:23:49,260 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b211e3aadde87eeeb10e4f0dad149691/A in TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:49,260 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/441d335b82014cd19a4de813ccc11a24, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/283a02a889d040e6bfe3c9950e72f3c1, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/f1769abfb5f04776884d7c9fb07e3d62, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/0b4348ac313144ba97de5e878c672caf] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp, totalSize=52.0 K 2024-11-08T00:23:49,261 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 441d335b82014cd19a4de813ccc11a24, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731025425760 2024-11-08T00:23:49,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b211e3aadde87eeeb10e4f0dad149691:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T00:23:49,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:23:49,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b211e3aadde87eeeb10e4f0dad149691:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:23:49,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:23:49,262 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 283a02a889d040e6bfe3c9950e72f3c1, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731025425908 2024-11-08T00:23:49,263 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46150 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T00:23:49,263 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1769abfb5f04776884d7c9fb07e3d62, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731025426643 2024-11-08T00:23:49,263 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): b211e3aadde87eeeb10e4f0dad149691/B is initiating minor compaction (all files) 2024-11-08T00:23:49,263 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b211e3aadde87eeeb10e4f0dad149691/B in TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:49,263 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/e2ae0a5be75d4e6ba65e0e36ada2966b, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/f28c43f5ab8444f6a7b91f495359a5f1, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/3c5be6f01f4e49f69b05148f93e8799e, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/9d8ca6499d774958846c2129dce08760] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp, totalSize=45.1 K 2024-11-08T00:23:49,263 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b4348ac313144ba97de5e878c672caf, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731025427422 2024-11-08T00:23:49,266 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting e2ae0a5be75d4e6ba65e0e36ada2966b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731025425760 2024-11-08T00:23:49,267 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting f28c43f5ab8444f6a7b91f495359a5f1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731025425910 2024-11-08T00:23:49,268 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c5be6f01f4e49f69b05148f93e8799e, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731025426643 2024-11-08T00:23:49,269 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d8ca6499d774958846c2129dce08760, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731025427430 2024-11-08T00:23:49,299 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b211e3aadde87eeeb10e4f0dad149691#A#compaction#33 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:23:49,300 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/7fe36e514feb410ebfdcb1d24e56f443 is 50, key is test_row_0/A:col10/1731025427574/Put/seqid=0 2024-11-08T00:23:49,318 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b211e3aadde87eeeb10e4f0dad149691#B#compaction#34 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:23:49,319 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/5209b6e937e2470faf7ecd725b34c000 is 50, key is test_row_0/B:col10/1731025427574/Put/seqid=0 2024-11-08T00:23:49,398 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-08T00:23:49,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741870_1046 (size=12527) 2024-11-08T00:23:49,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:49,398 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing b211e3aadde87eeeb10e4f0dad149691 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-08T00:23:49,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741871_1047 (size=12527) 2024-11-08T00:23:49,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/b711aae2c9dd4aebb02abf5220fdc26b is 50, key is test_row_1/A:col10/1731025427597/Put/seqid=0 2024-11-08T00:23:49,429 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/5209b6e937e2470faf7ecd725b34c000 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/5209b6e937e2470faf7ecd725b34c000 2024-11-08T00:23:49,429 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/7fe36e514feb410ebfdcb1d24e56f443 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/7fe36e514feb410ebfdcb1d24e56f443 2024-11-08T00:23:49,449 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b211e3aadde87eeeb10e4f0dad149691/B of b211e3aadde87eeeb10e4f0dad149691 into 5209b6e937e2470faf7ecd725b34c000(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:23:49,451 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b211e3aadde87eeeb10e4f0dad149691: 2024-11-08T00:23:49,451 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., storeName=b211e3aadde87eeeb10e4f0dad149691/B, priority=12, startTime=1731025429259; duration=0sec 2024-11-08T00:23:49,451 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:23:49,451 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b211e3aadde87eeeb10e4f0dad149691:B 2024-11-08T00:23:49,451 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T00:23:49,452 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b211e3aadde87eeeb10e4f0dad149691/A of b211e3aadde87eeeb10e4f0dad149691 into 7fe36e514feb410ebfdcb1d24e56f443(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:23:49,452 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b211e3aadde87eeeb10e4f0dad149691: 2024-11-08T00:23:49,452 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., storeName=b211e3aadde87eeeb10e4f0dad149691/A, priority=12, startTime=1731025429256; duration=0sec 2024-11-08T00:23:49,452 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:23:49,452 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b211e3aadde87eeeb10e4f0dad149691:A 2024-11-08T00:23:49,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741872_1048 (size=9757) 2024-11-08T00:23:49,458 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/b711aae2c9dd4aebb02abf5220fdc26b 2024-11-08T00:23:49,459 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46150 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T00:23:49,461 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): b211e3aadde87eeeb10e4f0dad149691/C is initiating minor compaction (all files) 2024-11-08T00:23:49,461 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b211e3aadde87eeeb10e4f0dad149691/C in TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:49,461 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/2fa338d8b64243efbf48b133750301fe, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/f5e4ace533af47d8969ae95395a87fa0, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/a82d8980c0ca4e598b8d282a6d79a4c0, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/ac8b7963794349d395f49cda6ce4d286] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp, totalSize=45.1 K 2024-11-08T00:23:49,463 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 2fa338d8b64243efbf48b133750301fe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731025425760 2024-11-08T00:23:49,465 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting f5e4ace533af47d8969ae95395a87fa0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731025425910 2024-11-08T00:23:49,466 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting a82d8980c0ca4e598b8d282a6d79a4c0, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731025426643 2024-11-08T00:23:49,468 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting ac8b7963794349d395f49cda6ce4d286, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731025427430 2024-11-08T00:23:49,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/f00910855c134b9aaa363aaefdeb61e5 is 50, key is test_row_1/B:col10/1731025427597/Put/seqid=0 2024-11-08T00:23:49,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741873_1049 (size=9757) 2024-11-08T00:23:49,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-08T00:23:49,530 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b211e3aadde87eeeb10e4f0dad149691#C#compaction#37 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:23:49,531 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/6be675b73d3f498fbbe0008b0af0eaac is 50, key is test_row_0/C:col10/1731025427574/Put/seqid=0 2024-11-08T00:23:49,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741874_1050 (size=12527) 2024-11-08T00:23:49,595 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/6be675b73d3f498fbbe0008b0af0eaac as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/6be675b73d3f498fbbe0008b0af0eaac 2024-11-08T00:23:49,610 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b211e3aadde87eeeb10e4f0dad149691/C of b211e3aadde87eeeb10e4f0dad149691 into 6be675b73d3f498fbbe0008b0af0eaac(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:23:49,610 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b211e3aadde87eeeb10e4f0dad149691: 2024-11-08T00:23:49,610 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., storeName=b211e3aadde87eeeb10e4f0dad149691/C, priority=12, startTime=1731025429262; duration=0sec 2024-11-08T00:23:49,611 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:23:49,611 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b211e3aadde87eeeb10e4f0dad149691:C 2024-11-08T00:23:49,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. as already flushing 2024-11-08T00:23:49,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on b211e3aadde87eeeb10e4f0dad149691 2024-11-08T00:23:49,923 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/f00910855c134b9aaa363aaefdeb61e5 2024-11-08T00:23:49,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/5eca041cd9eb429ebea96c91267041ab is 50, key is test_row_1/C:col10/1731025427597/Put/seqid=0 2024-11-08T00:23:49,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:49,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47486 deadline: 1731025489957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:49,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:49,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:49,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:49,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:49,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47520 deadline: 1731025489964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:49,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:49,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:49,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:49,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:49,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47524 deadline: 1731025489964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:49,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:49,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:49,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:50,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:50,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47466 deadline: 1731025489968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:50,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:50,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:50,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:50,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741875_1051 (size=9757) 2024-11-08T00:23:50,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:50,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47486 deadline: 1731025490068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:50,070 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:50,071 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:50,071 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:50,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:50,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47520 deadline: 1731025490081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:50,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:50,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:50,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:50,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:50,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47524 deadline: 1731025490088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:50,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:50,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:50,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:50,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:50,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47466 deadline: 1731025490133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:50,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:50,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:50,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:50,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:50,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47486 deadline: 1731025490281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:50,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:50,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:50,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:50,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:50,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47520 deadline: 1731025490301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:50,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:50,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:50,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47524 deadline: 1731025490301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:50,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:50,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:50,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:50,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:50,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:50,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:50,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47466 deadline: 1731025490348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:50,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:50,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:50,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:50,420 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/5eca041cd9eb429ebea96c91267041ab 2024-11-08T00:23:50,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/b711aae2c9dd4aebb02abf5220fdc26b as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/b711aae2c9dd4aebb02abf5220fdc26b 2024-11-08T00:23:50,457 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/b711aae2c9dd4aebb02abf5220fdc26b, entries=100, sequenceid=177, filesize=9.5 K 2024-11-08T00:23:50,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/f00910855c134b9aaa363aaefdeb61e5 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/f00910855c134b9aaa363aaefdeb61e5 2024-11-08T00:23:50,473 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/f00910855c134b9aaa363aaefdeb61e5, entries=100, sequenceid=177, filesize=9.5 K 2024-11-08T00:23:50,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/5eca041cd9eb429ebea96c91267041ab as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/5eca041cd9eb429ebea96c91267041ab 2024-11-08T00:23:50,492 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/5eca041cd9eb429ebea96c91267041ab, entries=100, sequenceid=177, filesize=9.5 K 2024-11-08T00:23:50,494 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=174.43 KB/178620 for b211e3aadde87eeeb10e4f0dad149691 in 1095ms, sequenceid=177, compaction requested=false 2024-11-08T00:23:50,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for b211e3aadde87eeeb10e4f0dad149691: 2024-11-08T00:23:50,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:50,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-08T00:23:50,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-08T00:23:50,500 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-08T00:23:50,500 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1080 sec 2024-11-08T00:23:50,503 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 2.1200 sec 2024-11-08T00:23:50,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-08T00:23:50,540 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:23:50,540 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-08T00:23:50,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-08T00:23:50,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-08T00:23:50,550 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T00:23:50,551 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T00:23:50,552 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T00:23:50,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on b211e3aadde87eeeb10e4f0dad149691 2024-11-08T00:23:50,612 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:23:50,613 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b211e3aadde87eeeb10e4f0dad149691 3/3 column families, dataSize=181.14 KB heapSize=475.36 KB 2024-11-08T00:23:50,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:50,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47524 deadline: 1731025490645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:50,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:50,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:50,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:50,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:50,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47520 deadline: 1731025490647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:50,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:50,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:50,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:50,655 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/7f912cb29b9c41b2a3d8949caf20ccc3 is 50, key is test_row_0/A:col10/1731025429960/Put/seqid=0 2024-11-08T00:23:50,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-08T00:23:50,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:50,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47486 deadline: 1731025490656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:50,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:50,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:50,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:50,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:50,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47466 deadline: 1731025490668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:50,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:50,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:50,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:50,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741876_1052 (size=12151) 2024-11-08T00:23:50,691 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/7f912cb29b9c41b2a3d8949caf20ccc3 2024-11-08T00:23:50,709 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-08T00:23:50,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:50,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. as already flushing 2024-11-08T00:23:50,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:50,710 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:50,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:50,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:50,724 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/190b95078c5b4a96bf6c1cb5387264c4 is 50, key is test_row_0/B:col10/1731025429960/Put/seqid=0 2024-11-08T00:23:50,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:50,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47524 deadline: 1731025490758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:50,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:50,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:50,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:50,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:50,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47520 deadline: 1731025490759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:50,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:50,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:50,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:50,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:50,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47486 deadline: 1731025490767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:50,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:50,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:50,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:50,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741877_1053 (size=12151) 2024-11-08T00:23:50,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-08T00:23:50,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:50,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. as already flushing 2024-11-08T00:23:50,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:50,865 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:50,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:50,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:50,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-08T00:23:50,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:50,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47524 deadline: 1731025490979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:50,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:50,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:50,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:50,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:50,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47520 deadline: 1731025490979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:50,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:50,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47486 deadline: 1731025490980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:50,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:50,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:50,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:50,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:50,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:50,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:51,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-08T00:23:51,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:51,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. as already flushing 2024-11-08T00:23:51,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:51,023 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:51,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:51,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:51,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-08T00:23:51,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-08T00:23:51,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:51,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. as already flushing 2024-11-08T00:23:51,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:51,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:51,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47466 deadline: 1731025491182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:51,188 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/190b95078c5b4a96bf6c1cb5387264c4 2024-11-08T00:23:51,196 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:51,184 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:51,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:51,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:51,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:51,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:51,260 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/e2ec732b48354270914a3382e9ba8339 is 50, key is test_row_0/C:col10/1731025429960/Put/seqid=0 2024-11-08T00:23:51,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:51,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47486 deadline: 1731025491298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:51,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:51,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:51,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:51,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:51,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47520 deadline: 1731025491301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:51,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:51,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:51,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:51,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:51,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47524 deadline: 1731025491301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:51,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:51,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:51,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:51,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741878_1054 (size=12151) 2024-11-08T00:23:51,336 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/e2ec732b48354270914a3382e9ba8339 2024-11-08T00:23:51,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-08T00:23:51,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:51,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. as already flushing 2024-11-08T00:23:51,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:51,359 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:51,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:51,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:51,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/7f912cb29b9c41b2a3d8949caf20ccc3 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/7f912cb29b9c41b2a3d8949caf20ccc3 2024-11-08T00:23:51,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/7f912cb29b9c41b2a3d8949caf20ccc3, entries=150, sequenceid=210, filesize=11.9 K 2024-11-08T00:23:51,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/190b95078c5b4a96bf6c1cb5387264c4 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/190b95078c5b4a96bf6c1cb5387264c4 2024-11-08T00:23:51,417 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/190b95078c5b4a96bf6c1cb5387264c4, entries=150, sequenceid=210, filesize=11.9 K 2024-11-08T00:23:51,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/e2ec732b48354270914a3382e9ba8339 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/e2ec732b48354270914a3382e9ba8339 2024-11-08T00:23:51,429 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/e2ec732b48354270914a3382e9ba8339, entries=150, sequenceid=210, filesize=11.9 K 2024-11-08T00:23:51,431 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=26.84 KB/27480 for b211e3aadde87eeeb10e4f0dad149691 in 818ms, sequenceid=210, compaction requested=true 2024-11-08T00:23:51,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b211e3aadde87eeeb10e4f0dad149691: 2024-11-08T00:23:51,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b211e3aadde87eeeb10e4f0dad149691:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T00:23:51,431 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:23:51,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:23:51,432 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:23:51,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b211e3aadde87eeeb10e4f0dad149691:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T00:23:51,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:23:51,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b211e3aadde87eeeb10e4f0dad149691:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:23:51,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:23:51,433 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34435 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:23:51,433 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): b211e3aadde87eeeb10e4f0dad149691/A is initiating minor compaction (all files) 2024-11-08T00:23:51,434 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b211e3aadde87eeeb10e4f0dad149691/A in TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:51,434 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/7fe36e514feb410ebfdcb1d24e56f443, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/b711aae2c9dd4aebb02abf5220fdc26b, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/7f912cb29b9c41b2a3d8949caf20ccc3] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp, totalSize=33.6 K 2024-11-08T00:23:51,434 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34435 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:23:51,434 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): b211e3aadde87eeeb10e4f0dad149691/B is initiating minor compaction (all files) 2024-11-08T00:23:51,435 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b211e3aadde87eeeb10e4f0dad149691/B in TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:51,435 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/5209b6e937e2470faf7ecd725b34c000, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/f00910855c134b9aaa363aaefdeb61e5, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/190b95078c5b4a96bf6c1cb5387264c4] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp, totalSize=33.6 K 2024-11-08T00:23:51,437 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7fe36e514feb410ebfdcb1d24e56f443, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731025427430 2024-11-08T00:23:51,438 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 5209b6e937e2470faf7ecd725b34c000, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731025427430 2024-11-08T00:23:51,438 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting b711aae2c9dd4aebb02abf5220fdc26b, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731025427597 2024-11-08T00:23:51,441 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting f00910855c134b9aaa363aaefdeb61e5, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731025427597 2024-11-08T00:23:51,442 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f912cb29b9c41b2a3d8949caf20ccc3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731025429960 2024-11-08T00:23:51,442 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 190b95078c5b4a96bf6c1cb5387264c4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731025429960 2024-11-08T00:23:51,482 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b211e3aadde87eeeb10e4f0dad149691#A#compaction#42 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:23:51,483 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/ec5cb1fa8b114181a4bc33b2a64eaec6 is 50, key is test_row_0/A:col10/1731025429960/Put/seqid=0 2024-11-08T00:23:51,491 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b211e3aadde87eeeb10e4f0dad149691#B#compaction#43 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:23:51,492 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/bae90d04dd544e399ced30caa2314b0d is 50, key is test_row_0/B:col10/1731025429960/Put/seqid=0 2024-11-08T00:23:51,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-08T00:23:51,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:51,521 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing b211e3aadde87eeeb10e4f0dad149691 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-08T00:23:51,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741879_1055 (size=12629) 2024-11-08T00:23:51,552 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/ec5cb1fa8b114181a4bc33b2a64eaec6 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/ec5cb1fa8b114181a4bc33b2a64eaec6 2024-11-08T00:23:51,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/75e26036921541b28b5a6877e8436b65 is 50, key is test_row_0/A:col10/1731025430645/Put/seqid=0 2024-11-08T00:23:51,566 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b211e3aadde87eeeb10e4f0dad149691/A of b211e3aadde87eeeb10e4f0dad149691 into ec5cb1fa8b114181a4bc33b2a64eaec6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:23:51,566 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b211e3aadde87eeeb10e4f0dad149691: 2024-11-08T00:23:51,567 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., storeName=b211e3aadde87eeeb10e4f0dad149691/A, priority=13, startTime=1731025431431; duration=0sec 2024-11-08T00:23:51,567 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:23:51,567 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b211e3aadde87eeeb10e4f0dad149691:A 2024-11-08T00:23:51,568 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:23:51,571 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34435 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:23:51,571 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): b211e3aadde87eeeb10e4f0dad149691/C is initiating minor compaction (all files) 2024-11-08T00:23:51,571 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b211e3aadde87eeeb10e4f0dad149691/C in TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:51,571 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/6be675b73d3f498fbbe0008b0af0eaac, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/5eca041cd9eb429ebea96c91267041ab, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/e2ec732b48354270914a3382e9ba8339] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp, totalSize=33.6 K 2024-11-08T00:23:51,572 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6be675b73d3f498fbbe0008b0af0eaac, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731025427430 2024-11-08T00:23:51,573 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5eca041cd9eb429ebea96c91267041ab, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731025427597 2024-11-08T00:23:51,574 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2ec732b48354270914a3382e9ba8339, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731025429960 2024-11-08T00:23:51,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741880_1056 (size=12629) 2024-11-08T00:23:51,601 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/bae90d04dd544e399ced30caa2314b0d as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/bae90d04dd544e399ced30caa2314b0d 2024-11-08T00:23:51,612 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b211e3aadde87eeeb10e4f0dad149691/B of b211e3aadde87eeeb10e4f0dad149691 into bae90d04dd544e399ced30caa2314b0d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:23:51,613 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b211e3aadde87eeeb10e4f0dad149691: 2024-11-08T00:23:51,613 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., storeName=b211e3aadde87eeeb10e4f0dad149691/B, priority=13, startTime=1731025431431; duration=0sec 2024-11-08T00:23:51,613 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:23:51,613 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b211e3aadde87eeeb10e4f0dad149691:B 2024-11-08T00:23:51,624 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b211e3aadde87eeeb10e4f0dad149691#C#compaction#45 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:23:51,624 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/8160d7b06c634c75bb93314c779c9a89 is 50, key is test_row_0/C:col10/1731025429960/Put/seqid=0 2024-11-08T00:23:51,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741881_1057 (size=9757) 2024-11-08T00:23:51,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-08T00:23:51,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741882_1058 (size=12629) 2024-11-08T00:23:51,723 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/8160d7b06c634c75bb93314c779c9a89 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/8160d7b06c634c75bb93314c779c9a89 2024-11-08T00:23:51,736 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b211e3aadde87eeeb10e4f0dad149691/C of b211e3aadde87eeeb10e4f0dad149691 into 8160d7b06c634c75bb93314c779c9a89(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:23:51,736 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b211e3aadde87eeeb10e4f0dad149691: 2024-11-08T00:23:51,736 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., storeName=b211e3aadde87eeeb10e4f0dad149691/C, priority=13, startTime=1731025431432; duration=0sec 2024-11-08T00:23:51,736 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:23:51,736 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b211e3aadde87eeeb10e4f0dad149691:C 2024-11-08T00:23:51,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on b211e3aadde87eeeb10e4f0dad149691 2024-11-08T00:23:51,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. as already flushing 2024-11-08T00:23:51,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:51,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47520 deadline: 1731025491944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:51,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:51,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:51,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:51,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:51,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47524 deadline: 1731025491943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:51,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:51,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:51,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:51,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:51,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47486 deadline: 1731025491952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:51,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:51,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:51,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:52,071 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/75e26036921541b28b5a6877e8436b65 2024-11-08T00:23:52,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:52,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47520 deadline: 1731025492058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:52,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:52,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:52,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:52,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:52,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47524 deadline: 1731025492068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:52,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:52,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:52,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:52,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:52,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47486 deadline: 1731025492070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:52,078 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:52,078 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:52,078 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:52,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/d3d53db3cd03469cbf5fdd8020f12b1d is 50, key is test_row_0/B:col10/1731025430645/Put/seqid=0 2024-11-08T00:23:52,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:52,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47492 deadline: 1731025492137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:52,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741883_1059 (size=9757) 2024-11-08T00:23:52,142 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 3302f0f507bd,40479,1731025415856 for 'test_row_1' in b211e3aadde87eeeb10e4f0dad149691 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8323 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T00:23:52,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:52,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:52,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:52,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:52,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47466 deadline: 1731025492207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:52,215 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 3302f0f507bd,40479,1731025415856 for 'test_row_1' in b211e3aadde87eeeb10e4f0dad149691 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2251 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T00:23:52,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:52,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:52,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:52,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:52,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47486 deadline: 1731025492289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:52,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:52,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:52,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:52,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:52,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:52,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47524 deadline: 1731025492289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:52,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47520 deadline: 1731025492289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:52,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:52,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:52,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:52,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:52,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:52,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:52,543 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/d3d53db3cd03469cbf5fdd8020f12b1d 2024-11-08T00:23:52,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/857d720996364130b698bf37b1703e5f is 50, key is test_row_0/C:col10/1731025430645/Put/seqid=0 2024-11-08T00:23:52,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741884_1060 (size=9757) 2024-11-08T00:23:52,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:52,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:52,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47524 deadline: 1731025492598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:52,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47486 deadline: 1731025492598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:52,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:52,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:52,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:52,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:52,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47520 deadline: 1731025492598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:52,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:52,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:52,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:52,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:52,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:52,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:52,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-08T00:23:53,004 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/857d720996364130b698bf37b1703e5f 2024-11-08T00:23:53,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/75e26036921541b28b5a6877e8436b65 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/75e26036921541b28b5a6877e8436b65 2024-11-08T00:23:53,061 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/75e26036921541b28b5a6877e8436b65, entries=100, sequenceid=217, filesize=9.5 K 2024-11-08T00:23:53,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/d3d53db3cd03469cbf5fdd8020f12b1d as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/d3d53db3cd03469cbf5fdd8020f12b1d 2024-11-08T00:23:53,079 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/d3d53db3cd03469cbf5fdd8020f12b1d, entries=100, sequenceid=217, filesize=9.5 K 2024-11-08T00:23:53,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/857d720996364130b698bf37b1703e5f as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/857d720996364130b698bf37b1703e5f 2024-11-08T00:23:53,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:53,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47524 deadline: 1731025493108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:53,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:53,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:53,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:53,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:53,116 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/857d720996364130b698bf37b1703e5f, entries=100, sequenceid=217, filesize=9.5 K 2024-11-08T00:23:53,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47520 deadline: 1731025493108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:53,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:53,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:53,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:53,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:53,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47486 deadline: 1731025493108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:53,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:53,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:53,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:53,125 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=174.43 KB/178620 for b211e3aadde87eeeb10e4f0dad149691 in 1604ms, sequenceid=217, compaction requested=false 2024-11-08T00:23:53,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for b211e3aadde87eeeb10e4f0dad149691: 2024-11-08T00:23:53,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:53,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-08T00:23:53,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-08T00:23:53,130 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-08T00:23:53,130 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5750 sec 2024-11-08T00:23:53,135 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 2.5910 sec 2024-11-08T00:23:54,131 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:23:54,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on b211e3aadde87eeeb10e4f0dad149691 2024-11-08T00:23:54,131 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b211e3aadde87eeeb10e4f0dad149691 3/3 column families, dataSize=181.14 KB heapSize=475.36 KB 2024-11-08T00:23:54,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:54,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47524 deadline: 1731025494134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:54,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:54,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:54,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:54,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:54,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47520 deadline: 1731025494142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:54,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:54,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:54,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:54,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:54,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47486 deadline: 1731025494143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:54,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:54,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:54,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:54,151 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/8336736154244353aa7cc3d5d1f6bb6a is 50, key is test_row_0/A:col10/1731025434129/Put/seqid=0 2024-11-08T00:23:54,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741885_1061 (size=14541) 2024-11-08T00:23:54,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:54,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47466 deadline: 1731025494227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:54,233 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 3302f0f507bd,40479,1731025415856 for 'test_row_1' in b211e3aadde87eeeb10e4f0dad149691 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4270 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T00:23:54,234 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:54,234 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:54,234 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:54,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:54,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47524 deadline: 1731025494248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:54,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:54,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47520 deadline: 1731025494249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:54,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:54,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:54,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:54,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:54,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:54,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:54,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:54,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47486 deadline: 1731025494258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:54,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:54,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:54,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:54,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:54,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47520 deadline: 1731025494457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:54,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:54,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:54,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:54,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:54,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47524 deadline: 1731025494468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:54,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:54,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:54,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:54,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:54,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47486 deadline: 1731025494468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:54,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:54,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:54,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:54,600 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/8336736154244353aa7cc3d5d1f6bb6a 2024-11-08T00:23:54,661 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/abce69cadf2e4d98a28107df8f370b3a is 50, key is test_row_0/B:col10/1731025434129/Put/seqid=0 2024-11-08T00:23:54,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741886_1062 (size=12151) 2024-11-08T00:23:54,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-08T00:23:54,725 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:23:54,733 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-08T00:23:54,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-08T00:23:54,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-08T00:23:54,742 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T00:23:54,744 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T00:23:54,744 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T00:23:54,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:54,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47520 deadline: 1731025494772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:54,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:54,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:54,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:54,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:54,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47524 deadline: 1731025494788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:54,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:54,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:54,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:54,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:54,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47486 deadline: 1731025494830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:54,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:54,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:54,837 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:54,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-08T00:23:54,900 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-08T00:23:54,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:54,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. as already flushing 2024-11-08T00:23:54,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:54,901 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:54,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:54,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:55,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-08T00:23:55,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-08T00:23:55,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:55,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. as already flushing 2024-11-08T00:23:55,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:55,058 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:55,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:55,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:55,110 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/abce69cadf2e4d98a28107df8f370b3a 2024-11-08T00:23:55,143 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/048757cf9b914a7caadb549dc0f7175f is 50, key is test_row_0/C:col10/1731025434129/Put/seqid=0 2024-11-08T00:23:55,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741887_1063 (size=12151) 2024-11-08T00:23:55,200 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/048757cf9b914a7caadb549dc0f7175f 2024-11-08T00:23:55,216 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-08T00:23:55,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:55,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. as already flushing 2024-11-08T00:23:55,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:55,220 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:55,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:55,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:55,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/8336736154244353aa7cc3d5d1f6bb6a as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/8336736154244353aa7cc3d5d1f6bb6a 2024-11-08T00:23:55,251 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/8336736154244353aa7cc3d5d1f6bb6a, entries=200, sequenceid=252, filesize=14.2 K 2024-11-08T00:23:55,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/abce69cadf2e4d98a28107df8f370b3a as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/abce69cadf2e4d98a28107df8f370b3a 2024-11-08T00:23:55,283 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/abce69cadf2e4d98a28107df8f370b3a, entries=150, sequenceid=252, filesize=11.9 K 2024-11-08T00:23:55,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/048757cf9b914a7caadb549dc0f7175f as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/048757cf9b914a7caadb549dc0f7175f 2024-11-08T00:23:55,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:55,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47520 deadline: 1731025495292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:55,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:55,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:55,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:55,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:55,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47524 deadline: 1731025495308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:55,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:55,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:55,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:55,335 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/048757cf9b914a7caadb549dc0f7175f, entries=150, sequenceid=252, filesize=11.9 K 2024-11-08T00:23:55,343 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~194.56 KB/199230, heapSize ~510.47 KB/522720, currentSize=13.42 KB/13740 for b211e3aadde87eeeb10e4f0dad149691 in 1212ms, sequenceid=252, compaction requested=true 2024-11-08T00:23:55,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b211e3aadde87eeeb10e4f0dad149691: 2024-11-08T00:23:55,343 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:23:55,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b211e3aadde87eeeb10e4f0dad149691:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T00:23:55,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:23:55,344 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:23:55,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b211e3aadde87eeeb10e4f0dad149691:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T00:23:55,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:23:55,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b211e3aadde87eeeb10e4f0dad149691:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:23:55,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:23:55,349 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36927 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:23:55,349 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): b211e3aadde87eeeb10e4f0dad149691/A is initiating minor compaction (all files) 2024-11-08T00:23:55,349 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b211e3aadde87eeeb10e4f0dad149691/A in TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:55,349 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/ec5cb1fa8b114181a4bc33b2a64eaec6, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/75e26036921541b28b5a6877e8436b65, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/8336736154244353aa7cc3d5d1f6bb6a] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp, totalSize=36.1 K 2024-11-08T00:23:55,350 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec5cb1fa8b114181a4bc33b2a64eaec6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731025429960 2024-11-08T00:23:55,351 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34537 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:23:55,351 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): b211e3aadde87eeeb10e4f0dad149691/B is initiating minor compaction (all files) 2024-11-08T00:23:55,351 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b211e3aadde87eeeb10e4f0dad149691/B in TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:55,351 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/bae90d04dd544e399ced30caa2314b0d, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/d3d53db3cd03469cbf5fdd8020f12b1d, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/abce69cadf2e4d98a28107df8f370b3a] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp, totalSize=33.7 K 2024-11-08T00:23:55,351 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75e26036921541b28b5a6877e8436b65, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731025430630 2024-11-08T00:23:55,352 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting bae90d04dd544e399ced30caa2314b0d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731025429960 2024-11-08T00:23:55,353 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8336736154244353aa7cc3d5d1f6bb6a, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731025431943 2024-11-08T00:23:55,355 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting d3d53db3cd03469cbf5fdd8020f12b1d, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731025430630 2024-11-08T00:23:55,356 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting abce69cadf2e4d98a28107df8f370b3a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731025431943 2024-11-08T00:23:55,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-08T00:23:55,381 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-08T00:23:55,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:55,382 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing b211e3aadde87eeeb10e4f0dad149691 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-08T00:23:55,382 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b211e3aadde87eeeb10e4f0dad149691#B#compaction#51 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:23:55,387 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b211e3aadde87eeeb10e4f0dad149691#A#compaction#52 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:23:55,387 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/944c0f6fe850433181571b1b153b8dc0 is 50, key is test_row_0/B:col10/1731025434129/Put/seqid=0 2024-11-08T00:23:55,387 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/f4fbae6264f0415cb6ab90196ef06a4f is 50, key is test_row_0/A:col10/1731025434129/Put/seqid=0 2024-11-08T00:23:55,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/9d9199e43d974ad5bf77b42025a0bbb3 is 50, key is test_row_0/A:col10/1731025435368/Put/seqid=0 2024-11-08T00:23:55,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on b211e3aadde87eeeb10e4f0dad149691 2024-11-08T00:23:55,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. as already flushing 2024-11-08T00:23:55,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741889_1065 (size=12731) 2024-11-08T00:23:55,479 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/f4fbae6264f0415cb6ab90196ef06a4f as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/f4fbae6264f0415cb6ab90196ef06a4f 2024-11-08T00:23:55,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741888_1064 (size=12731) 2024-11-08T00:23:55,504 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b211e3aadde87eeeb10e4f0dad149691/A of b211e3aadde87eeeb10e4f0dad149691 into f4fbae6264f0415cb6ab90196ef06a4f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:23:55,504 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b211e3aadde87eeeb10e4f0dad149691: 2024-11-08T00:23:55,504 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., storeName=b211e3aadde87eeeb10e4f0dad149691/A, priority=13, startTime=1731025435343; duration=0sec 2024-11-08T00:23:55,504 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:23:55,504 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b211e3aadde87eeeb10e4f0dad149691:A 2024-11-08T00:23:55,504 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:23:55,507 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/944c0f6fe850433181571b1b153b8dc0 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/944c0f6fe850433181571b1b153b8dc0 2024-11-08T00:23:55,507 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34537 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:23:55,507 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): b211e3aadde87eeeb10e4f0dad149691/C is initiating minor compaction (all files) 2024-11-08T00:23:55,507 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b211e3aadde87eeeb10e4f0dad149691/C in TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:55,508 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/8160d7b06c634c75bb93314c779c9a89, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/857d720996364130b698bf37b1703e5f, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/048757cf9b914a7caadb549dc0f7175f] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp, totalSize=33.7 K 2024-11-08T00:23:55,509 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8160d7b06c634c75bb93314c779c9a89, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731025429960 2024-11-08T00:23:55,510 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 857d720996364130b698bf37b1703e5f, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731025430630 2024-11-08T00:23:55,510 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 048757cf9b914a7caadb549dc0f7175f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731025431943 2024-11-08T00:23:55,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741890_1066 (size=14691) 2024-11-08T00:23:55,518 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/9d9199e43d974ad5bf77b42025a0bbb3 2024-11-08T00:23:55,531 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b211e3aadde87eeeb10e4f0dad149691/B of b211e3aadde87eeeb10e4f0dad149691 into 944c0f6fe850433181571b1b153b8dc0(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:23:55,531 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b211e3aadde87eeeb10e4f0dad149691: 2024-11-08T00:23:55,532 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., storeName=b211e3aadde87eeeb10e4f0dad149691/B, priority=13, startTime=1731025435344; duration=0sec 2024-11-08T00:23:55,532 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:23:55,532 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b211e3aadde87eeeb10e4f0dad149691:B 2024-11-08T00:23:55,539 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b211e3aadde87eeeb10e4f0dad149691#C#compaction#54 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:23:55,540 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/d9f4bd1aa8f5441eace0e8319782c8a9 is 50, key is test_row_0/C:col10/1731025434129/Put/seqid=0 2024-11-08T00:23:55,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/b41eb4738ebe42cf817856c8e774524d is 50, key is test_row_0/B:col10/1731025435368/Put/seqid=0 2024-11-08T00:23:55,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741892_1068 (size=12251) 2024-11-08T00:23:55,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741891_1067 (size=12731) 2024-11-08T00:23:55,639 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/d9f4bd1aa8f5441eace0e8319782c8a9 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/d9f4bd1aa8f5441eace0e8319782c8a9 2024-11-08T00:23:55,671 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b211e3aadde87eeeb10e4f0dad149691/C of b211e3aadde87eeeb10e4f0dad149691 into d9f4bd1aa8f5441eace0e8319782c8a9(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:23:55,671 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b211e3aadde87eeeb10e4f0dad149691: 2024-11-08T00:23:55,671 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., storeName=b211e3aadde87eeeb10e4f0dad149691/C, priority=13, startTime=1731025435344; duration=0sec 2024-11-08T00:23:55,671 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:23:55,671 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b211e3aadde87eeeb10e4f0dad149691:C 2024-11-08T00:23:55,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:55,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47486 deadline: 1731025495767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:55,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:55,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:55,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:55,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-08T00:23:55,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:55,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47486 deadline: 1731025495881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:55,883 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:55,883 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:55,883 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:55,983 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/b41eb4738ebe42cf817856c8e774524d 2024-11-08T00:23:56,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/1ee4468c62954d29935af3046f1380a9 is 50, key is test_row_0/C:col10/1731025435368/Put/seqid=0 2024-11-08T00:23:56,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741893_1069 (size=12251) 2024-11-08T00:23:56,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:56,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47486 deadline: 1731025496088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:56,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:56,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:56,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:56,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:56,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47520 deadline: 1731025496307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:56,312 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3302f0f507bd,40479,1731025415856 for 'test_row_0' in b211e3aadde87eeeb10e4f0dad149691 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T00:23:56,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:56,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:56,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:56,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:56,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47524 deadline: 1731025496328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:56,330 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 3302f0f507bd,40479,1731025415856 for 'test_row_2' in b211e3aadde87eeeb10e4f0dad149691 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T00:23:56,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:56,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:56,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:56,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:56,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47486 deadline: 1731025496397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:56,399 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:56,399 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:56,399 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:56,436 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/1ee4468c62954d29935af3046f1380a9 2024-11-08T00:23:56,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/9d9199e43d974ad5bf77b42025a0bbb3 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/9d9199e43d974ad5bf77b42025a0bbb3 2024-11-08T00:23:56,457 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/9d9199e43d974ad5bf77b42025a0bbb3, entries=200, sequenceid=261, filesize=14.3 K 2024-11-08T00:23:56,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/b41eb4738ebe42cf817856c8e774524d as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/b41eb4738ebe42cf817856c8e774524d 2024-11-08T00:23:56,490 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/b41eb4738ebe42cf817856c8e774524d, entries=150, sequenceid=261, filesize=12.0 K 2024-11-08T00:23:56,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/1ee4468c62954d29935af3046f1380a9 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/1ee4468c62954d29935af3046f1380a9 2024-11-08T00:23:56,515 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/1ee4468c62954d29935af3046f1380a9, entries=150, sequenceid=261, filesize=12.0 K 2024-11-08T00:23:56,517 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for b211e3aadde87eeeb10e4f0dad149691 in 1136ms, sequenceid=261, compaction requested=false 2024-11-08T00:23:56,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for b211e3aadde87eeeb10e4f0dad149691: 2024-11-08T00:23:56,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:56,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-08T00:23:56,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-08T00:23:56,524 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-08T00:23:56,525 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7770 sec 2024-11-08T00:23:56,529 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.7920 sec 2024-11-08T00:23:56,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-08T00:23:56,888 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:23:56,890 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-08T00:23:56,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-08T00:23:56,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-08T00:23:56,893 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T00:23:56,894 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T00:23:56,894 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T00:23:56,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on b211e3aadde87eeeb10e4f0dad149691 2024-11-08T00:23:56,910 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:23:56,910 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b211e3aadde87eeeb10e4f0dad149691 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-08T00:23:56,920 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/f581ed99fdc044d4b13364cdb424b26f is 50, key is test_row_0/A:col10/1731025435699/Put/seqid=0 2024-11-08T00:23:56,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741894_1070 (size=14741) 2024-11-08T00:23:56,938 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/f581ed99fdc044d4b13364cdb424b26f 2024-11-08T00:23:56,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:56,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47486 deadline: 1731025496943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:56,950 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:56,950 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:56,950 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:56,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/cc9b14e5d465482fb8e1e708668296eb is 50, key is test_row_0/B:col10/1731025435699/Put/seqid=0 2024-11-08T00:23:56,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741895_1071 (size=12301) 2024-11-08T00:23:56,995 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/cc9b14e5d465482fb8e1e708668296eb 2024-11-08T00:23:57,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-08T00:23:57,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/364aba4804234a38bd99d0336f5614c9 is 50, key is test_row_0/C:col10/1731025435699/Put/seqid=0 2024-11-08T00:23:57,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-08T00:23:57,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:57,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. as already flushing 2024-11-08T00:23:57,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:57,051 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:57,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:57,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:23:57,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:57,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47486 deadline: 1731025497058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:57,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:57,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:57,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:57,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741896_1072 (size=12301) 2024-11-08T00:23:57,082 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/364aba4804234a38bd99d0336f5614c9 2024-11-08T00:23:57,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/f581ed99fdc044d4b13364cdb424b26f as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/f581ed99fdc044d4b13364cdb424b26f 2024-11-08T00:23:57,110 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/f581ed99fdc044d4b13364cdb424b26f, entries=200, sequenceid=292, filesize=14.4 K 2024-11-08T00:23:57,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/cc9b14e5d465482fb8e1e708668296eb as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/cc9b14e5d465482fb8e1e708668296eb 2024-11-08T00:23:57,129 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/cc9b14e5d465482fb8e1e708668296eb, entries=150, sequenceid=292, filesize=12.0 K 2024-11-08T00:23:57,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/364aba4804234a38bd99d0336f5614c9 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/364aba4804234a38bd99d0336f5614c9 2024-11-08T00:23:57,141 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/364aba4804234a38bd99d0336f5614c9, entries=150, sequenceid=292, filesize=12.0 K 2024-11-08T00:23:57,144 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for b211e3aadde87eeeb10e4f0dad149691 in 234ms, sequenceid=292, compaction requested=true 2024-11-08T00:23:57,144 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b211e3aadde87eeeb10e4f0dad149691: 2024-11-08T00:23:57,144 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:23:57,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b211e3aadde87eeeb10e4f0dad149691:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T00:23:57,146 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42163 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:23:57,146 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): b211e3aadde87eeeb10e4f0dad149691/A is initiating minor compaction (all files) 2024-11-08T00:23:57,146 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b211e3aadde87eeeb10e4f0dad149691/A in TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:57,146 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/f4fbae6264f0415cb6ab90196ef06a4f, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/9d9199e43d974ad5bf77b42025a0bbb3, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/f581ed99fdc044d4b13364cdb424b26f] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp, totalSize=41.2 K 2024-11-08T00:23:57,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:23:57,147 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4fbae6264f0415cb6ab90196ef06a4f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731025431943 2024-11-08T00:23:57,147 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:23:57,147 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d9199e43d974ad5bf77b42025a0bbb3, keycount=200, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731025434140 2024-11-08T00:23:57,148 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37283 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:23:57,148 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): b211e3aadde87eeeb10e4f0dad149691/B is initiating minor compaction (all files) 2024-11-08T00:23:57,148 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b211e3aadde87eeeb10e4f0dad149691/B in TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:57,149 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/944c0f6fe850433181571b1b153b8dc0, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/b41eb4738ebe42cf817856c8e774524d, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/cc9b14e5d465482fb8e1e708668296eb] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp, totalSize=36.4 K 2024-11-08T00:23:57,149 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting f581ed99fdc044d4b13364cdb424b26f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731025435699 2024-11-08T00:23:57,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b211e3aadde87eeeb10e4f0dad149691:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T00:23:57,150 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 944c0f6fe850433181571b1b153b8dc0, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731025431943 2024-11-08T00:23:57,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:23:57,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b211e3aadde87eeeb10e4f0dad149691:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:23:57,151 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting b41eb4738ebe42cf817856c8e774524d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731025434140 2024-11-08T00:23:57,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:23:57,151 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting cc9b14e5d465482fb8e1e708668296eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731025435699 2024-11-08T00:23:57,168 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b211e3aadde87eeeb10e4f0dad149691#A#compaction#60 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:23:57,169 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/79502a1467ac426a919295bd5bfca51d is 50, key is test_row_0/A:col10/1731025435699/Put/seqid=0 2024-11-08T00:23:57,187 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b211e3aadde87eeeb10e4f0dad149691#B#compaction#61 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:23:57,188 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/d67f52b2905b4eeaacf851e06de189fe is 50, key is test_row_0/B:col10/1731025435699/Put/seqid=0 2024-11-08T00:23:57,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-08T00:23:57,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-08T00:23:57,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:57,213 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing b211e3aadde87eeeb10e4f0dad149691 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-08T00:23:57,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/10be81670dbf48e980a91a3a3674afd2 is 50, key is test_row_0/A:col10/1731025436932/Put/seqid=0 2024-11-08T00:23:57,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741897_1073 (size=12983) 2024-11-08T00:23:57,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741898_1074 (size=12983) 2024-11-08T00:23:57,266 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/79502a1467ac426a919295bd5bfca51d as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/79502a1467ac426a919295bd5bfca51d 2024-11-08T00:23:57,277 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/d67f52b2905b4eeaacf851e06de189fe as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/d67f52b2905b4eeaacf851e06de189fe 2024-11-08T00:23:57,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741899_1075 (size=12301) 2024-11-08T00:23:57,282 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/10be81670dbf48e980a91a3a3674afd2 2024-11-08T00:23:57,289 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b211e3aadde87eeeb10e4f0dad149691/A of b211e3aadde87eeeb10e4f0dad149691 into 79502a1467ac426a919295bd5bfca51d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:23:57,312 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b211e3aadde87eeeb10e4f0dad149691: 2024-11-08T00:23:57,312 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., storeName=b211e3aadde87eeeb10e4f0dad149691/A, priority=13, startTime=1731025437144; duration=0sec 2024-11-08T00:23:57,312 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b211e3aadde87eeeb10e4f0dad149691/B of b211e3aadde87eeeb10e4f0dad149691 into d67f52b2905b4eeaacf851e06de189fe(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:23:57,312 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b211e3aadde87eeeb10e4f0dad149691: 2024-11-08T00:23:57,312 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:23:57,312 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., storeName=b211e3aadde87eeeb10e4f0dad149691/B, priority=13, startTime=1731025437147; duration=0sec 2024-11-08T00:23:57,312 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b211e3aadde87eeeb10e4f0dad149691:A 2024-11-08T00:23:57,312 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:23:57,312 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:23:57,312 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b211e3aadde87eeeb10e4f0dad149691:B 2024-11-08T00:23:57,315 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37283 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:23:57,315 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): b211e3aadde87eeeb10e4f0dad149691/C is initiating minor compaction (all files) 2024-11-08T00:23:57,315 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b211e3aadde87eeeb10e4f0dad149691/C in TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:57,316 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/d9f4bd1aa8f5441eace0e8319782c8a9, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/1ee4468c62954d29935af3046f1380a9, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/364aba4804234a38bd99d0336f5614c9] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp, totalSize=36.4 K 2024-11-08T00:23:57,317 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9f4bd1aa8f5441eace0e8319782c8a9, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731025431943 2024-11-08T00:23:57,319 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ee4468c62954d29935af3046f1380a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731025434140 2024-11-08T00:23:57,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/179316b8bb1941eba33a8fd4d1e56978 is 50, key is test_row_0/B:col10/1731025436932/Put/seqid=0 2024-11-08T00:23:57,319 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 364aba4804234a38bd99d0336f5614c9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731025435699 2024-11-08T00:23:57,334 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b211e3aadde87eeeb10e4f0dad149691#C#compaction#64 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:23:57,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. as already flushing 2024-11-08T00:23:57,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741900_1076 (size=12301) 2024-11-08T00:23:57,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on b211e3aadde87eeeb10e4f0dad149691 2024-11-08T00:23:57,335 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/6d5cd66872a24ce4a2519b451835a4d1 is 50, key is test_row_0/C:col10/1731025435699/Put/seqid=0 2024-11-08T00:23:57,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741901_1077 (size=12983) 2024-11-08T00:23:57,464 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/6d5cd66872a24ce4a2519b451835a4d1 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/6d5cd66872a24ce4a2519b451835a4d1 2024-11-08T00:23:57,477 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b211e3aadde87eeeb10e4f0dad149691/C of b211e3aadde87eeeb10e4f0dad149691 into 6d5cd66872a24ce4a2519b451835a4d1(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:23:57,477 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b211e3aadde87eeeb10e4f0dad149691: 2024-11-08T00:23:57,477 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., storeName=b211e3aadde87eeeb10e4f0dad149691/C, priority=13, startTime=1731025437151; duration=0sec 2024-11-08T00:23:57,477 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:23:57,477 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b211e3aadde87eeeb10e4f0dad149691:C 2024-11-08T00:23:57,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-08T00:23:57,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:57,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47486 deadline: 1731025497554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:57,560 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:57,560 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:57,560 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:57,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:57,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47486 deadline: 1731025497667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:57,673 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:57,673 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:57,673 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:57,736 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/179316b8bb1941eba33a8fd4d1e56978 2024-11-08T00:23:57,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/befee3af3a21497fa8d0d7698c47a7b7 is 50, key is test_row_0/C:col10/1731025436932/Put/seqid=0 2024-11-08T00:23:57,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741902_1078 (size=12301) 2024-11-08T00:23:57,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:57,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47486 deadline: 1731025497877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:57,883 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:57,883 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:57,883 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:58,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-08T00:23:58,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:58,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47486 deadline: 1731025498187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:58,190 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:58,190 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:58,190 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:58,222 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/befee3af3a21497fa8d0d7698c47a7b7 2024-11-08T00:23:58,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/10be81670dbf48e980a91a3a3674afd2 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/10be81670dbf48e980a91a3a3674afd2 2024-11-08T00:23:58,252 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/10be81670dbf48e980a91a3a3674afd2, entries=150, sequenceid=300, filesize=12.0 K 2024-11-08T00:23:58,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/179316b8bb1941eba33a8fd4d1e56978 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/179316b8bb1941eba33a8fd4d1e56978 2024-11-08T00:23:58,263 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/179316b8bb1941eba33a8fd4d1e56978, entries=150, sequenceid=300, filesize=12.0 K 2024-11-08T00:23:58,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:58,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47466 deadline: 1731025498257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:58,265 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 3302f0f507bd,40479,1731025415856 for 'test_row_1' in b211e3aadde87eeeb10e4f0dad149691 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8301 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T00:23:58,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:58,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:58,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:58,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/befee3af3a21497fa8d0d7698c47a7b7 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/befee3af3a21497fa8d0d7698c47a7b7 2024-11-08T00:23:58,278 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/befee3af3a21497fa8d0d7698c47a7b7, entries=150, sequenceid=300, filesize=12.0 K 2024-11-08T00:23:58,281 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=167.72 KB/171750 for b211e3aadde87eeeb10e4f0dad149691 in 1068ms, sequenceid=300, compaction requested=false 2024-11-08T00:23:58,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for b211e3aadde87eeeb10e4f0dad149691: 2024-11-08T00:23:58,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:58,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-08T00:23:58,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-08T00:23:58,290 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-08T00:23:58,290 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3920 sec 2024-11-08T00:23:58,301 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.4020 sec 2024-11-08T00:23:58,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on b211e3aadde87eeeb10e4f0dad149691 2024-11-08T00:23:58,364 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:23:58,364 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b211e3aadde87eeeb10e4f0dad149691 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-08T00:23:58,380 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/cbc31695e2d043eeb657dd02d25f9edc is 50, key is test_row_0/A:col10/1731025438352/Put/seqid=0 2024-11-08T00:23:58,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:58,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47524 deadline: 1731025498386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:58,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:58,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47520 deadline: 1731025498387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:58,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:58,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:58,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:58,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:58,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:58,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:58,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741903_1079 (size=14741) 2024-11-08T00:23:58,434 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/cbc31695e2d043eeb657dd02d25f9edc 2024-11-08T00:23:58,463 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/e3c73a74d66341a298325370d5c4e49f is 50, key is test_row_0/B:col10/1731025438352/Put/seqid=0 2024-11-08T00:23:58,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:58,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47524 deadline: 1731025498497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:58,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:58,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:58,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:58,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741904_1080 (size=12301) 2024-11-08T00:23:58,509 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/e3c73a74d66341a298325370d5c4e49f 2024-11-08T00:23:58,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:58,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47520 deadline: 1731025498518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:58,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:58,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:58,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:58,531 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/3ab11fbb644e41e08007a66a894ac5de is 50, key is test_row_0/C:col10/1731025438352/Put/seqid=0 2024-11-08T00:23:58,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741905_1081 (size=12301) 2024-11-08T00:23:58,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:58,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47486 deadline: 1731025498698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:58,701 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:58,701 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:58,701 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:58,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:58,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47524 deadline: 1731025498707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:58,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:58,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:58,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:58,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:58,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47520 deadline: 1731025498728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:58,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:58,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:58,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:58,987 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/3ab11fbb644e41e08007a66a894ac5de 2024-11-08T00:23:58,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/cbc31695e2d043eeb657dd02d25f9edc as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/cbc31695e2d043eeb657dd02d25f9edc 2024-11-08T00:23:59,011 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/cbc31695e2d043eeb657dd02d25f9edc, entries=200, sequenceid=333, filesize=14.4 K 2024-11-08T00:23:59,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/e3c73a74d66341a298325370d5c4e49f as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/e3c73a74d66341a298325370d5c4e49f 2024-11-08T00:23:59,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:59,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47524 deadline: 1731025499018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:59,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:59,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:59,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:59,028 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/e3c73a74d66341a298325370d5c4e49f, entries=150, sequenceid=333, filesize=12.0 K 2024-11-08T00:23:59,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/3ab11fbb644e41e08007a66a894ac5de as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/3ab11fbb644e41e08007a66a894ac5de 2024-11-08T00:23:59,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-08T00:23:59,038 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:23:59,039 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-08T00:23:59,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:59,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47520 deadline: 1731025499037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:59,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:59,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:59,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:59,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-08T00:23:59,043 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T00:23:59,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-08T00:23:59,044 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T00:23:59,044 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T00:23:59,066 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/3ab11fbb644e41e08007a66a894ac5de, entries=150, sequenceid=333, filesize=12.0 K 2024-11-08T00:23:59,068 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=20.13 KB/20610 for b211e3aadde87eeeb10e4f0dad149691 in 703ms, sequenceid=333, compaction requested=true 2024-11-08T00:23:59,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b211e3aadde87eeeb10e4f0dad149691: 2024-11-08T00:23:59,068 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:23:59,069 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:23:59,070 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): b211e3aadde87eeeb10e4f0dad149691/A is initiating minor compaction (all files) 2024-11-08T00:23:59,070 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b211e3aadde87eeeb10e4f0dad149691/A in TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:59,070 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/79502a1467ac426a919295bd5bfca51d, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/10be81670dbf48e980a91a3a3674afd2, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/cbc31695e2d043eeb657dd02d25f9edc] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp, totalSize=39.1 K 2024-11-08T00:23:59,071 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79502a1467ac426a919295bd5bfca51d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731025435699 2024-11-08T00:23:59,071 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10be81670dbf48e980a91a3a3674afd2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731025436925 2024-11-08T00:23:59,075 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbc31695e2d043eeb657dd02d25f9edc, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731025437515 2024-11-08T00:23:59,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b211e3aadde87eeeb10e4f0dad149691:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T00:23:59,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:23:59,089 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:23:59,091 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:23:59,091 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): b211e3aadde87eeeb10e4f0dad149691/B is initiating minor compaction (all files) 2024-11-08T00:23:59,091 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b211e3aadde87eeeb10e4f0dad149691/B in TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:59,091 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/d67f52b2905b4eeaacf851e06de189fe, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/179316b8bb1941eba33a8fd4d1e56978, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/e3c73a74d66341a298325370d5c4e49f] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp, totalSize=36.7 K 2024-11-08T00:23:59,092 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting d67f52b2905b4eeaacf851e06de189fe, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731025435699 2024-11-08T00:23:59,093 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 179316b8bb1941eba33a8fd4d1e56978, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731025436925 2024-11-08T00:23:59,093 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting e3c73a74d66341a298325370d5c4e49f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731025437548 2024-11-08T00:23:59,102 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b211e3aadde87eeeb10e4f0dad149691#A#compaction#69 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:23:59,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b211e3aadde87eeeb10e4f0dad149691:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T00:23:59,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:23:59,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b211e3aadde87eeeb10e4f0dad149691:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:23:59,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:23:59,103 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/c6c0ae2b5533458f90ce9c4952199d45 is 50, key is test_row_0/A:col10/1731025438352/Put/seqid=0 2024-11-08T00:23:59,105 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b211e3aadde87eeeb10e4f0dad149691#B#compaction#70 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:23:59,106 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/ced4a99db97a4aa3836e244fddc57cd2 is 50, key is test_row_0/B:col10/1731025438352/Put/seqid=0 2024-11-08T00:23:59,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-08T00:23:59,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741906_1082 (size=13085) 2024-11-08T00:23:59,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741907_1083 (size=13085) 2024-11-08T00:23:59,178 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/c6c0ae2b5533458f90ce9c4952199d45 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/c6c0ae2b5533458f90ce9c4952199d45 2024-11-08T00:23:59,189 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b211e3aadde87eeeb10e4f0dad149691/A of b211e3aadde87eeeb10e4f0dad149691 into c6c0ae2b5533458f90ce9c4952199d45(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:23:59,189 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b211e3aadde87eeeb10e4f0dad149691: 2024-11-08T00:23:59,189 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., storeName=b211e3aadde87eeeb10e4f0dad149691/A, priority=13, startTime=1731025439068; duration=0sec 2024-11-08T00:23:59,189 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:23:59,189 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b211e3aadde87eeeb10e4f0dad149691:A 2024-11-08T00:23:59,189 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:23:59,191 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:23:59,191 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): b211e3aadde87eeeb10e4f0dad149691/C is initiating minor compaction (all files) 2024-11-08T00:23:59,191 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b211e3aadde87eeeb10e4f0dad149691/C in TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:59,192 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/6d5cd66872a24ce4a2519b451835a4d1, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/befee3af3a21497fa8d0d7698c47a7b7, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/3ab11fbb644e41e08007a66a894ac5de] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp, totalSize=36.7 K 2024-11-08T00:23:59,192 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d5cd66872a24ce4a2519b451835a4d1, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731025435699 2024-11-08T00:23:59,198 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting befee3af3a21497fa8d0d7698c47a7b7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731025436925 2024-11-08T00:23:59,198 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ab11fbb644e41e08007a66a894ac5de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731025437548 2024-11-08T00:23:59,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-08T00:23:59,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:59,201 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/ced4a99db97a4aa3836e244fddc57cd2 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/ced4a99db97a4aa3836e244fddc57cd2 2024-11-08T00:23:59,201 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing b211e3aadde87eeeb10e4f0dad149691 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-08T00:23:59,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/523e55cf3e0c4c9fa876fcb6eedf1da7 is 50, key is test_row_0/A:col10/1731025438373/Put/seqid=0 2024-11-08T00:23:59,227 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b211e3aadde87eeeb10e4f0dad149691/B of b211e3aadde87eeeb10e4f0dad149691 into ced4a99db97a4aa3836e244fddc57cd2(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:23:59,227 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b211e3aadde87eeeb10e4f0dad149691: 2024-11-08T00:23:59,227 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., storeName=b211e3aadde87eeeb10e4f0dad149691/B, priority=13, startTime=1731025439089; duration=0sec 2024-11-08T00:23:59,227 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:23:59,227 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b211e3aadde87eeeb10e4f0dad149691:B 2024-11-08T00:23:59,252 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b211e3aadde87eeeb10e4f0dad149691#C#compaction#72 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:23:59,253 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/00c229c532be4ceabb372b9045845c7e is 50, key is test_row_0/C:col10/1731025438352/Put/seqid=0 2024-11-08T00:23:59,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741908_1084 (size=12301) 2024-11-08T00:23:59,285 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/523e55cf3e0c4c9fa876fcb6eedf1da7 2024-11-08T00:23:59,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/06b49c5fda3b464aab0a26004d35296a is 50, key is test_row_0/B:col10/1731025438373/Put/seqid=0 2024-11-08T00:23:59,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741909_1085 (size=13085) 2024-11-08T00:23:59,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741910_1086 (size=12301) 2024-11-08T00:23:59,353 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/06b49c5fda3b464aab0a26004d35296a 2024-11-08T00:23:59,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-08T00:23:59,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/00d8776ac832491d9317b9b01df042ce is 50, key is test_row_0/C:col10/1731025438373/Put/seqid=0 2024-11-08T00:23:59,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741911_1087 (size=12301) 2024-11-08T00:23:59,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. as already flushing 2024-11-08T00:23:59,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on b211e3aadde87eeeb10e4f0dad149691 2024-11-08T00:23:59,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-08T00:23:59,736 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/00c229c532be4ceabb372b9045845c7e as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/00c229c532be4ceabb372b9045845c7e 2024-11-08T00:23:59,744 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b211e3aadde87eeeb10e4f0dad149691/C of b211e3aadde87eeeb10e4f0dad149691 into 00c229c532be4ceabb372b9045845c7e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:23:59,744 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b211e3aadde87eeeb10e4f0dad149691: 2024-11-08T00:23:59,745 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., storeName=b211e3aadde87eeeb10e4f0dad149691/C, priority=13, startTime=1731025439103; duration=0sec 2024-11-08T00:23:59,745 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:23:59,745 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b211e3aadde87eeeb10e4f0dad149691:C 2024-11-08T00:23:59,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:59,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47486 deadline: 1731025499750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:59,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:59,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:59,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:59,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:59,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47524 deadline: 1731025499754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:59,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:59,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:59,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:59,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:59,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47520 deadline: 1731025499771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:59,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:59,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:59,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:59,799 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/00d8776ac832491d9317b9b01df042ce 2024-11-08T00:23:59,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/523e55cf3e0c4c9fa876fcb6eedf1da7 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/523e55cf3e0c4c9fa876fcb6eedf1da7 2024-11-08T00:23:59,820 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/523e55cf3e0c4c9fa876fcb6eedf1da7, entries=150, sequenceid=340, filesize=12.0 K 2024-11-08T00:23:59,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/06b49c5fda3b464aab0a26004d35296a as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/06b49c5fda3b464aab0a26004d35296a 2024-11-08T00:23:59,834 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/06b49c5fda3b464aab0a26004d35296a, entries=150, sequenceid=340, filesize=12.0 K 2024-11-08T00:23:59,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/00d8776ac832491d9317b9b01df042ce as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/00d8776ac832491d9317b9b01df042ce 2024-11-08T00:23:59,852 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/00d8776ac832491d9317b9b01df042ce, entries=150, sequenceid=340, filesize=12.0 K 2024-11-08T00:23:59,854 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=181.14 KB/185490 for b211e3aadde87eeeb10e4f0dad149691 in 652ms, sequenceid=340, compaction requested=false 2024-11-08T00:23:59,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for b211e3aadde87eeeb10e4f0dad149691: 2024-11-08T00:23:59,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:23:59,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-08T00:23:59,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-08T00:23:59,867 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-08T00:23:59,867 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 820 msec 2024-11-08T00:23:59,872 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 831 msec 2024-11-08T00:23:59,894 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:23:59,894 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b211e3aadde87eeeb10e4f0dad149691 3/3 column families, dataSize=194.56 KB heapSize=510.52 KB 2024-11-08T00:23:59,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on b211e3aadde87eeeb10e4f0dad149691 2024-11-08T00:23:59,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:59,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47486 deadline: 1731025499896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:59,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:59,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:59,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:59,917 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/e71ad53bea91438c9eb4a42a00e714e6 is 50, key is test_row_0/A:col10/1731025439690/Put/seqid=0 2024-11-08T00:23:59,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:59,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:23:59,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47520 deadline: 1731025499900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:59,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47524 deadline: 1731025499898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:23:59,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:59,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:59,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:59,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:59,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:23:59,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:23:59,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741912_1088 (size=17181) 2024-11-08T00:24:00,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:00,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47520 deadline: 1731025500033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:00,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:00,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47486 deadline: 1731025500033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:00,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:00,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:00,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:00,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:00,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47524 deadline: 1731025500036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:00,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:00,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:00,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:00,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:00,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:00,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:00,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-08T00:24:00,178 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:24:00,178 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-08T00:24:00,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-08T00:24:00,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-08T00:24:00,187 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T00:24:00,188 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T00:24:00,188 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T00:24:00,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:00,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47520 deadline: 1731025500248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:00,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:00,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:00,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:00,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:00,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47486 deadline: 1731025500258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:00,271 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:00,271 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:00,271 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:00,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:00,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47524 deadline: 1731025500268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:00,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:00,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:00,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:00,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-08T00:24:00,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-08T00:24:00,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:24:00,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. as already flushing 2024-11-08T00:24:00,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:24:00,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:00,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:00,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:00,359 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/e71ad53bea91438c9eb4a42a00e714e6 2024-11-08T00:24:00,394 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/b6e699f6350d43f2a0014c796f80031d is 50, key is test_row_0/B:col10/1731025439690/Put/seqid=0 2024-11-08T00:24:00,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741913_1089 (size=12301) 2024-11-08T00:24:00,417 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/b6e699f6350d43f2a0014c796f80031d 2024-11-08T00:24:00,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/6eeece1245d24f408819adfe600e3379 is 50, key is test_row_0/C:col10/1731025439690/Put/seqid=0 2024-11-08T00:24:00,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741914_1090 (size=12301) 2024-11-08T00:24:00,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-08T00:24:00,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-08T00:24:00,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:24:00,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. as already flushing 2024-11-08T00:24:00,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:24:00,507 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:00,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:00,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:00,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:00,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47520 deadline: 1731025500557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:00,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:00,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:00,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:00,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:00,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47486 deadline: 1731025500579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:00,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:00,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:00,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:00,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:00,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47524 deadline: 1731025500585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:00,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:00,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:00,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:00,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-08T00:24:00,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:24:00,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. as already flushing 2024-11-08T00:24:00,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:24:00,664 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:00,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:00,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:00,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-08T00:24:00,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-08T00:24:00,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:24:00,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. as already flushing 2024-11-08T00:24:00,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:24:00,819 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:00,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:00,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:00,871 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/6eeece1245d24f408819adfe600e3379 2024-11-08T00:24:00,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/e71ad53bea91438c9eb4a42a00e714e6 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/e71ad53bea91438c9eb4a42a00e714e6 2024-11-08T00:24:00,892 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/e71ad53bea91438c9eb4a42a00e714e6, entries=250, sequenceid=374, filesize=16.8 K 2024-11-08T00:24:00,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/b6e699f6350d43f2a0014c796f80031d as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/b6e699f6350d43f2a0014c796f80031d 2024-11-08T00:24:00,907 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/b6e699f6350d43f2a0014c796f80031d, entries=150, sequenceid=374, filesize=12.0 K 2024-11-08T00:24:00,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/6eeece1245d24f408819adfe600e3379 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/6eeece1245d24f408819adfe600e3379 2024-11-08T00:24:00,926 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/6eeece1245d24f408819adfe600e3379, entries=150, sequenceid=374, filesize=12.0 K 2024-11-08T00:24:00,928 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~194.56 KB/199230, heapSize ~510.47 KB/522720, currentSize=6.71 KB/6870 for b211e3aadde87eeeb10e4f0dad149691 in 1033ms, sequenceid=374, compaction requested=true 2024-11-08T00:24:00,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b211e3aadde87eeeb10e4f0dad149691: 2024-11-08T00:24:00,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b211e3aadde87eeeb10e4f0dad149691:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T00:24:00,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:00,928 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:24:00,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b211e3aadde87eeeb10e4f0dad149691:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T00:24:00,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:24:00,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b211e3aadde87eeeb10e4f0dad149691:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:24:00,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-08T00:24:00,928 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:24:00,930 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42567 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:24:00,931 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): b211e3aadde87eeeb10e4f0dad149691/A is initiating minor compaction (all files) 2024-11-08T00:24:00,931 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b211e3aadde87eeeb10e4f0dad149691/A in TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:24:00,931 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/c6c0ae2b5533458f90ce9c4952199d45, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/523e55cf3e0c4c9fa876fcb6eedf1da7, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/e71ad53bea91438c9eb4a42a00e714e6] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp, totalSize=41.6 K 2024-11-08T00:24:00,931 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:24:00,931 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): b211e3aadde87eeeb10e4f0dad149691/B is initiating minor compaction (all files) 2024-11-08T00:24:00,931 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b211e3aadde87eeeb10e4f0dad149691/B in TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:24:00,932 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/ced4a99db97a4aa3836e244fddc57cd2, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/06b49c5fda3b464aab0a26004d35296a, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/b6e699f6350d43f2a0014c796f80031d] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp, totalSize=36.8 K 2024-11-08T00:24:00,932 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6c0ae2b5533458f90ce9c4952199d45, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731025437548 2024-11-08T00:24:00,932 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting ced4a99db97a4aa3836e244fddc57cd2, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731025437548 2024-11-08T00:24:00,932 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 523e55cf3e0c4c9fa876fcb6eedf1da7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731025438373 2024-11-08T00:24:00,932 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 06b49c5fda3b464aab0a26004d35296a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731025438373 2024-11-08T00:24:00,933 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting e71ad53bea91438c9eb4a42a00e714e6, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1731025439690 2024-11-08T00:24:00,935 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting b6e699f6350d43f2a0014c796f80031d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1731025439690 2024-11-08T00:24:00,957 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b211e3aadde87eeeb10e4f0dad149691#A#compaction#78 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:00,957 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/862766db44dd4a529a90184bfbff0751 is 50, key is test_row_0/A:col10/1731025439690/Put/seqid=0 2024-11-08T00:24:00,960 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b211e3aadde87eeeb10e4f0dad149691#B#compaction#79 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:00,961 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/70c454d5a5a54433a1895abb09f4fd69 is 50, key is test_row_0/B:col10/1731025439690/Put/seqid=0 2024-11-08T00:24:00,975 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-08T00:24:00,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:24:00,975 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing b211e3aadde87eeeb10e4f0dad149691 3/3 column families, dataSize=6.71 KB heapSize=18.33 KB 2024-11-08T00:24:00,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741915_1091 (size=13187) 2024-11-08T00:24:00,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741916_1092 (size=13187) 2024-11-08T00:24:00,988 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/70c454d5a5a54433a1895abb09f4fd69 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/70c454d5a5a54433a1895abb09f4fd69 2024-11-08T00:24:00,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/862301e3db324648acf11363ae0daca7 is 50, key is test_row_1/A:col10/1731025439896/Put/seqid=0 2024-11-08T00:24:01,005 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b211e3aadde87eeeb10e4f0dad149691/B of b211e3aadde87eeeb10e4f0dad149691 into 70c454d5a5a54433a1895abb09f4fd69(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:01,006 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b211e3aadde87eeeb10e4f0dad149691: 2024-11-08T00:24:01,006 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., storeName=b211e3aadde87eeeb10e4f0dad149691/B, priority=13, startTime=1731025440928; duration=0sec 2024-11-08T00:24:01,006 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:24:01,006 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b211e3aadde87eeeb10e4f0dad149691:B 2024-11-08T00:24:01,006 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:24:01,008 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:24:01,008 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): b211e3aadde87eeeb10e4f0dad149691/C is initiating minor compaction (all files) 2024-11-08T00:24:01,008 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b211e3aadde87eeeb10e4f0dad149691/C in TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:24:01,008 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/00c229c532be4ceabb372b9045845c7e, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/00d8776ac832491d9317b9b01df042ce, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/6eeece1245d24f408819adfe600e3379] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp, totalSize=36.8 K 2024-11-08T00:24:01,009 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 00c229c532be4ceabb372b9045845c7e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731025437548 2024-11-08T00:24:01,010 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 00d8776ac832491d9317b9b01df042ce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731025438373 2024-11-08T00:24:01,010 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 6eeece1245d24f408819adfe600e3379, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1731025439690 2024-11-08T00:24:01,018 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/862766db44dd4a529a90184bfbff0751 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/862766db44dd4a529a90184bfbff0751 2024-11-08T00:24:01,036 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b211e3aadde87eeeb10e4f0dad149691#C#compaction#81 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:01,037 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/ee1f46ff47c147b682905f87d466d2ff is 50, key is test_row_0/C:col10/1731025439690/Put/seqid=0 2024-11-08T00:24:01,037 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b211e3aadde87eeeb10e4f0dad149691/A of b211e3aadde87eeeb10e4f0dad149691 into 862766db44dd4a529a90184bfbff0751(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:01,037 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b211e3aadde87eeeb10e4f0dad149691: 2024-11-08T00:24:01,037 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., storeName=b211e3aadde87eeeb10e4f0dad149691/A, priority=13, startTime=1731025440928; duration=0sec 2024-11-08T00:24:01,037 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:01,037 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b211e3aadde87eeeb10e4f0dad149691:A 2024-11-08T00:24:01,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741917_1093 (size=7415) 2024-11-08T00:24:01,062 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/862301e3db324648acf11363ae0daca7 2024-11-08T00:24:01,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741918_1094 (size=13187) 2024-11-08T00:24:01,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/dc2cff7f1b32484e8e78e7f5e9c0d8b4 is 50, key is test_row_1/B:col10/1731025439896/Put/seqid=0 2024-11-08T00:24:01,095 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/ee1f46ff47c147b682905f87d466d2ff as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/ee1f46ff47c147b682905f87d466d2ff 2024-11-08T00:24:01,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. as already flushing 2024-11-08T00:24:01,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on b211e3aadde87eeeb10e4f0dad149691 2024-11-08T00:24:01,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741919_1095 (size=7415) 2024-11-08T00:24:01,104 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/dc2cff7f1b32484e8e78e7f5e9c0d8b4 2024-11-08T00:24:01,119 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b211e3aadde87eeeb10e4f0dad149691/C of b211e3aadde87eeeb10e4f0dad149691 into ee1f46ff47c147b682905f87d466d2ff(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:01,119 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b211e3aadde87eeeb10e4f0dad149691: 2024-11-08T00:24:01,120 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., storeName=b211e3aadde87eeeb10e4f0dad149691/C, priority=13, startTime=1731025440928; duration=0sec 2024-11-08T00:24:01,120 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:01,120 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b211e3aadde87eeeb10e4f0dad149691:C 2024-11-08T00:24:01,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/145c1ebe6c6b4dc2bddaf2473d28d87a is 50, key is test_row_1/C:col10/1731025439896/Put/seqid=0 2024-11-08T00:24:01,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741920_1096 (size=7415) 2024-11-08T00:24:01,135 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/145c1ebe6c6b4dc2bddaf2473d28d87a 2024-11-08T00:24:01,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/862301e3db324648acf11363ae0daca7 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/862301e3db324648acf11363ae0daca7 2024-11-08T00:24:01,160 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/862301e3db324648acf11363ae0daca7, entries=50, sequenceid=378, filesize=7.2 K 2024-11-08T00:24:01,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/dc2cff7f1b32484e8e78e7f5e9c0d8b4 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/dc2cff7f1b32484e8e78e7f5e9c0d8b4 2024-11-08T00:24:01,178 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/dc2cff7f1b32484e8e78e7f5e9c0d8b4, entries=50, sequenceid=378, filesize=7.2 K 2024-11-08T00:24:01,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/145c1ebe6c6b4dc2bddaf2473d28d87a as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/145c1ebe6c6b4dc2bddaf2473d28d87a 2024-11-08T00:24:01,195 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/145c1ebe6c6b4dc2bddaf2473d28d87a, entries=50, sequenceid=378, filesize=7.2 K 2024-11-08T00:24:01,296 INFO [AsyncFSWAL-0-hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f-prefix:3302f0f507bd,40479,1731025415856 {}] wal.AbstractFSWAL(1368): Slow sync cost: 107 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39671,DS-be635dc0-6437-4ba0-b479-97b2f6ad84a4,DISK]] 2024-11-08T00:24:01,296 INFO [AsyncFSWAL-0-hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f-prefix:3302f0f507bd,40479,1731025415856 {}] wal.AbstractFSWAL(1368): Slow sync cost: 101 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39671,DS-be635dc0-6437-4ba0-b479-97b2f6ad84a4,DISK]] 2024-11-08T00:24:01,297 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=181.14 KB/185490 for b211e3aadde87eeeb10e4f0dad149691 in 321ms, sequenceid=378, compaction requested=false 2024-11-08T00:24:01,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for b211e3aadde87eeeb10e4f0dad149691: 2024-11-08T00:24:01,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:24:01,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-08T00:24:01,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on b211e3aadde87eeeb10e4f0dad149691 2024-11-08T00:24:01,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-08T00:24:01,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:01,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47524 deadline: 1731025501299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:01,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:01,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47520 deadline: 1731025501299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:01,302 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:24:01,302 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b211e3aadde87eeeb10e4f0dad149691 3/3 column families, dataSize=201.27 KB heapSize=528.09 KB 2024-11-08T00:24:01,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:01,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:01,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:01,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:01,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:01,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:01,305 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-08T00:24:01,305 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1110 sec 2024-11-08T00:24:01,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:01,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47486 deadline: 1731025501304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:01,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:01,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:01,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:01,311 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.1310 sec 2024-11-08T00:24:01,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/ccb4e5978a9b4d6898b6e21f206b8cde is 50, key is test_row_0/A:col10/1731025441189/Put/seqid=0 2024-11-08T00:24:01,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-08T00:24:01,327 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:24:01,328 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-08T00:24:01,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-08T00:24:01,331 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T00:24:01,333 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T00:24:01,333 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T00:24:01,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-08T00:24:01,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741921_1097 (size=12301) 2024-11-08T00:24:01,367 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.09 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/ccb4e5978a9b4d6898b6e21f206b8cde 2024-11-08T00:24:01,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/cfa002d815224d159e82bea1157bab01 is 50, key is test_row_0/B:col10/1731025441189/Put/seqid=0 2024-11-08T00:24:01,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:01,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47520 deadline: 1731025501430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:01,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:01,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47486 deadline: 1731025501431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:01,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:01,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47524 deadline: 1731025501433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:01,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:01,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:01,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:01,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:01,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:01,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:01,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-08T00:24:01,442 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:01,442 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:01,442 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:01,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741922_1098 (size=12301) 2024-11-08T00:24:01,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-08T00:24:01,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:24:01,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. as already flushing 2024-11-08T00:24:01,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:24:01,489 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:01,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:01,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:01,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:01,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47524 deadline: 1731025501638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:01,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:01,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:01,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:01,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-08T00:24:01,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-08T00:24:01,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:24:01,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. as already flushing 2024-11-08T00:24:01,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:24:01,650 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:01,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:01,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:01,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47486 deadline: 1731025501647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:01,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:01,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:01,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47520 deadline: 1731025501648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:01,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:01,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:01,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:01,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:01,654 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:01,654 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:01,805 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-08T00:24:01,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:24:01,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. as already flushing 2024-11-08T00:24:01,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:24:01,805 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:01,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:01,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:01,887 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-08T00:24:01,888 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T00:24:01,891 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-08T00:24:01,891 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-08T00:24:01,891 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T00:24:01,891 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T00:24:01,893 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-08T00:24:01,893 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T00:24:01,894 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:24:01,894 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:24:01,894 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:24:01,895 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:24:01,895 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:24:01,895 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:24:01,895 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:24:01,896 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:24:01,896 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:24:01,896 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:24:01,897 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:24:01,897 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:24:01,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:01,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47524 deadline: 1731025501947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:01,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:01,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:01,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:01,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-08T00:24:01,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:01,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:01,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47520 deadline: 1731025501957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:01,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47486 deadline: 1731025501958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:01,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:01,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:01,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:01,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:01,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:01,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:01,959 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-08T00:24:01,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:24:01,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. as already flushing 2024-11-08T00:24:01,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:24:01,959 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:01,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:01,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:02,101 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.09 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/cfa002d815224d159e82bea1157bab01 2024-11-08T00:24:02,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-08T00:24:02,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:24:02,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. as already flushing 2024-11-08T00:24:02,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:24:02,113 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:02,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:02,113 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/daeec0e384d742aea86bd17aea5e7697 is 50, key is test_row_0/C:col10/1731025441189/Put/seqid=0 2024-11-08T00:24:02,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:02,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741923_1099 (size=12301) 2024-11-08T00:24:02,132 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.09 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/daeec0e384d742aea86bd17aea5e7697 2024-11-08T00:24:02,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/ccb4e5978a9b4d6898b6e21f206b8cde as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/ccb4e5978a9b4d6898b6e21f206b8cde 2024-11-08T00:24:02,158 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/ccb4e5978a9b4d6898b6e21f206b8cde, entries=150, sequenceid=414, filesize=12.0 K 2024-11-08T00:24:02,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/cfa002d815224d159e82bea1157bab01 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/cfa002d815224d159e82bea1157bab01 2024-11-08T00:24:02,168 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/cfa002d815224d159e82bea1157bab01, entries=150, sequenceid=414, filesize=12.0 K 2024-11-08T00:24:02,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/daeec0e384d742aea86bd17aea5e7697 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/daeec0e384d742aea86bd17aea5e7697 2024-11-08T00:24:02,180 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/daeec0e384d742aea86bd17aea5e7697, entries=150, sequenceid=414, filesize=12.0 K 2024-11-08T00:24:02,181 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~201.27 KB/206100, heapSize ~528.05 KB/540720, currentSize=0 B/0 for b211e3aadde87eeeb10e4f0dad149691 in 879ms, sequenceid=414, compaction requested=true 2024-11-08T00:24:02,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b211e3aadde87eeeb10e4f0dad149691: 2024-11-08T00:24:02,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b211e3aadde87eeeb10e4f0dad149691:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T00:24:02,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:02,182 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:24:02,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b211e3aadde87eeeb10e4f0dad149691:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T00:24:02,182 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:24:02,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:02,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b211e3aadde87eeeb10e4f0dad149691:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:24:02,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:24:02,183 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 32903 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:24:02,183 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 32903 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:24:02,183 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): b211e3aadde87eeeb10e4f0dad149691/B is initiating minor compaction (all files) 2024-11-08T00:24:02,183 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): b211e3aadde87eeeb10e4f0dad149691/A is initiating minor compaction (all files) 2024-11-08T00:24:02,183 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b211e3aadde87eeeb10e4f0dad149691/B in TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:24:02,183 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b211e3aadde87eeeb10e4f0dad149691/A in TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:24:02,183 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/70c454d5a5a54433a1895abb09f4fd69, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/dc2cff7f1b32484e8e78e7f5e9c0d8b4, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/cfa002d815224d159e82bea1157bab01] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp, totalSize=32.1 K 2024-11-08T00:24:02,183 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/862766db44dd4a529a90184bfbff0751, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/862301e3db324648acf11363ae0daca7, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/ccb4e5978a9b4d6898b6e21f206b8cde] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp, totalSize=32.1 K 2024-11-08T00:24:02,184 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 70c454d5a5a54433a1895abb09f4fd69, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1731025439690 2024-11-08T00:24:02,184 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 862766db44dd4a529a90184bfbff0751, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1731025439690 2024-11-08T00:24:02,184 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting dc2cff7f1b32484e8e78e7f5e9c0d8b4, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1731025439896 2024-11-08T00:24:02,184 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting cfa002d815224d159e82bea1157bab01, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1731025441185 2024-11-08T00:24:02,184 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 862301e3db324648acf11363ae0daca7, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1731025439896 2024-11-08T00:24:02,185 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting ccb4e5978a9b4d6898b6e21f206b8cde, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1731025441185 2024-11-08T00:24:02,202 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b211e3aadde87eeeb10e4f0dad149691#B#compaction#87 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:02,203 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/256950ef1fa94fabae806de30ceed13a is 50, key is test_row_0/B:col10/1731025441189/Put/seqid=0 2024-11-08T00:24:02,207 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b211e3aadde87eeeb10e4f0dad149691#A#compaction#88 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:02,208 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/1bef880d258b44b1ab7047fa99e26d2d is 50, key is test_row_0/A:col10/1731025441189/Put/seqid=0 2024-11-08T00:24:02,226 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-08T00:24:02,226 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T00:24:02,226 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:24:02,227 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:24:02,227 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:24:02,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741924_1100 (size=13289) 2024-11-08T00:24:02,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741925_1101 (size=13289) 2024-11-08T00:24:02,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-08T00:24:02,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:24:02,268 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing b211e3aadde87eeeb10e4f0dad149691 3/3 column families, dataSize=6.71 KB heapSize=18.33 KB 2024-11-08T00:24:02,276 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/1bef880d258b44b1ab7047fa99e26d2d as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/1bef880d258b44b1ab7047fa99e26d2d 2024-11-08T00:24:02,285 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b211e3aadde87eeeb10e4f0dad149691/A of b211e3aadde87eeeb10e4f0dad149691 into 1bef880d258b44b1ab7047fa99e26d2d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:02,285 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b211e3aadde87eeeb10e4f0dad149691: 2024-11-08T00:24:02,285 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., storeName=b211e3aadde87eeeb10e4f0dad149691/A, priority=13, startTime=1731025442181; duration=0sec 2024-11-08T00:24:02,285 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:24:02,285 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b211e3aadde87eeeb10e4f0dad149691:A 2024-11-08T00:24:02,285 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:24:02,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/b8b7501df5cd4484b8cda7c57bb27e67 is 50, key is test_row_1/A:col10/1731025442224/Put/seqid=0 2024-11-08T00:24:02,287 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 32903 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:24:02,287 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): b211e3aadde87eeeb10e4f0dad149691/C is initiating minor compaction (all files) 2024-11-08T00:24:02,288 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b211e3aadde87eeeb10e4f0dad149691/C in TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:24:02,288 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/ee1f46ff47c147b682905f87d466d2ff, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/145c1ebe6c6b4dc2bddaf2473d28d87a, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/daeec0e384d742aea86bd17aea5e7697] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp, totalSize=32.1 K 2024-11-08T00:24:02,289 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee1f46ff47c147b682905f87d466d2ff, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1731025439690 2024-11-08T00:24:02,290 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 145c1ebe6c6b4dc2bddaf2473d28d87a, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1731025439896 2024-11-08T00:24:02,290 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting daeec0e384d742aea86bd17aea5e7697, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1731025441185 2024-11-08T00:24:02,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741926_1102 (size=7415) 2024-11-08T00:24:02,318 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b211e3aadde87eeeb10e4f0dad149691#C#compaction#90 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:02,318 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/ebecb70781594987a96c65469ee5379c is 50, key is test_row_0/C:col10/1731025441189/Put/seqid=0 2024-11-08T00:24:02,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741927_1103 (size=13289) 2024-11-08T00:24:02,461 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-08T00:24:02,462 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T00:24:02,462 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:24:02,462 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:24:02,462 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:24:02,471 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-08T00:24:02,472 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T00:24:02,472 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:24:02,472 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:24:02,472 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:24:02,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-08T00:24:02,472 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-08T00:24:02,473 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T00:24:02,473 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:24:02,473 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:24:02,473 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:24:02,666 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/256950ef1fa94fabae806de30ceed13a as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/256950ef1fa94fabae806de30ceed13a 2024-11-08T00:24:02,675 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b211e3aadde87eeeb10e4f0dad149691/B of b211e3aadde87eeeb10e4f0dad149691 into 256950ef1fa94fabae806de30ceed13a(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:02,675 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b211e3aadde87eeeb10e4f0dad149691: 2024-11-08T00:24:02,675 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., storeName=b211e3aadde87eeeb10e4f0dad149691/B, priority=13, startTime=1731025442182; duration=0sec 2024-11-08T00:24:02,675 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:02,675 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b211e3aadde87eeeb10e4f0dad149691:B 2024-11-08T00:24:02,712 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/b8b7501df5cd4484b8cda7c57bb27e67 2024-11-08T00:24:02,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/3c6e0fcff4234ba3bfbe87a1b1164a5f is 50, key is test_row_1/B:col10/1731025442224/Put/seqid=0 2024-11-08T00:24:02,745 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/ebecb70781594987a96c65469ee5379c as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/ebecb70781594987a96c65469ee5379c 2024-11-08T00:24:02,762 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b211e3aadde87eeeb10e4f0dad149691/C of b211e3aadde87eeeb10e4f0dad149691 into ebecb70781594987a96c65469ee5379c(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:02,762 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b211e3aadde87eeeb10e4f0dad149691: 2024-11-08T00:24:02,762 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., storeName=b211e3aadde87eeeb10e4f0dad149691/C, priority=13, startTime=1731025442182; duration=0sec 2024-11-08T00:24:02,762 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:02,762 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b211e3aadde87eeeb10e4f0dad149691:C 2024-11-08T00:24:02,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741928_1104 (size=7415) 2024-11-08T00:24:02,782 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/3c6e0fcff4234ba3bfbe87a1b1164a5f 2024-11-08T00:24:02,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/50182074d73a4cd997c2e58eb4b31cdb is 50, key is test_row_1/C:col10/1731025442224/Put/seqid=0 2024-11-08T00:24:02,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741929_1105 (size=7415) 2024-11-08T00:24:02,809 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/50182074d73a4cd997c2e58eb4b31cdb 2024-11-08T00:24:02,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/b8b7501df5cd4484b8cda7c57bb27e67 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/b8b7501df5cd4484b8cda7c57bb27e67 2024-11-08T00:24:02,828 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/b8b7501df5cd4484b8cda7c57bb27e67, entries=50, sequenceid=418, filesize=7.2 K 2024-11-08T00:24:02,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/3c6e0fcff4234ba3bfbe87a1b1164a5f as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/3c6e0fcff4234ba3bfbe87a1b1164a5f 2024-11-08T00:24:02,839 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/3c6e0fcff4234ba3bfbe87a1b1164a5f, entries=50, sequenceid=418, filesize=7.2 K 2024-11-08T00:24:02,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/50182074d73a4cd997c2e58eb4b31cdb as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/50182074d73a4cd997c2e58eb4b31cdb 2024-11-08T00:24:02,848 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/50182074d73a4cd997c2e58eb4b31cdb, entries=50, sequenceid=418, filesize=7.2 K 2024-11-08T00:24:02,851 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=20.13 KB/20610 for b211e3aadde87eeeb10e4f0dad149691 in 583ms, sequenceid=418, compaction requested=false 2024-11-08T00:24:02,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for b211e3aadde87eeeb10e4f0dad149691: 2024-11-08T00:24:02,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:24:02,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-08T00:24:02,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-08T00:24:02,857 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-08T00:24:02,857 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5210 sec 2024-11-08T00:24:02,860 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 1.5300 sec 2024-11-08T00:24:02,917 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T00:24:03,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-08T00:24:03,493 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:24:08,370 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-08T00:24:08,370 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T00:24:08,370 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:24:08,370 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:24:08,371 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:24:08,372 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-08T00:24:08,372 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 122 2024-11-08T00:24:08,372 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 13 2024-11-08T00:24:08,372 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 77 2024-11-08T00:24:08,372 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 83 2024-11-08T00:24:08,372 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 36 2024-11-08T00:24:08,372 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-08T00:24:08,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2024 2024-11-08T00:24:08,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2017 2024-11-08T00:24:08,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-08T00:24:08,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1041 2024-11-08T00:24:08,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3122 rows 2024-11-08T00:24:08,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1036 2024-11-08T00:24:08,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3095 rows 2024-11-08T00:24:08,373 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T00:24:08,373 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T00:24:08,373 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:24:08,373 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:24:08,380 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:24:08,386 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-08T00:24:08,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-08T00:24:08,397 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731025448396"}]},"ts":"1731025448396"} 2024-11-08T00:24:08,401 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-08T00:24:08,401 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-08T00:24:08,403 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-08T00:24:08,413 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b211e3aadde87eeeb10e4f0dad149691, UNASSIGN}] 2024-11-08T00:24:08,415 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b211e3aadde87eeeb10e4f0dad149691, UNASSIGN 2024-11-08T00:24:08,417 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=b211e3aadde87eeeb10e4f0dad149691, regionState=CLOSING, regionLocation=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:08,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-08T00:24:08,425 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b211e3aadde87eeeb10e4f0dad149691, UNASSIGN because future has completed 2024-11-08T00:24:08,432 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-08T00:24:08,433 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856}] 2024-11-08T00:24:08,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-08T00:24:08,596 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(122): Close b211e3aadde87eeeb10e4f0dad149691 2024-11-08T00:24:08,596 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-08T00:24:08,597 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1722): Closing b211e3aadde87eeeb10e4f0dad149691, disabling compactions & flushes 2024-11-08T00:24:08,597 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:24:08,597 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:24:08,597 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. after waiting 0 ms 2024-11-08T00:24:08,597 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:24:08,597 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(2902): Flushing b211e3aadde87eeeb10e4f0dad149691 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-08T00:24:08,603 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/f080dc006e0f4be7af98f941e88d7289 is 50, key is test_row_1/A:col10/1731025448368/Put/seqid=0 2024-11-08T00:24:08,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741930_1106 (size=9857) 2024-11-08T00:24:08,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-08T00:24:09,014 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/f080dc006e0f4be7af98f941e88d7289 2024-11-08T00:24:09,025 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/7965ad7c01134eb7bb15de5d414f08ef is 50, key is test_row_1/B:col10/1731025448368/Put/seqid=0 2024-11-08T00:24:09,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741931_1107 (size=9857) 2024-11-08T00:24:09,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-08T00:24:09,437 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/7965ad7c01134eb7bb15de5d414f08ef 2024-11-08T00:24:09,470 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/c413a16c45b249d7b4781e35b6a62253 is 50, key is test_row_1/C:col10/1731025448368/Put/seqid=0 2024-11-08T00:24:09,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741932_1108 (size=9857) 2024-11-08T00:24:09,494 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/c413a16c45b249d7b4781e35b6a62253 2024-11-08T00:24:09,503 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/A/f080dc006e0f4be7af98f941e88d7289 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/f080dc006e0f4be7af98f941e88d7289 2024-11-08T00:24:09,510 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/f080dc006e0f4be7af98f941e88d7289, entries=100, sequenceid=428, filesize=9.6 K 2024-11-08T00:24:09,512 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/B/7965ad7c01134eb7bb15de5d414f08ef as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/7965ad7c01134eb7bb15de5d414f08ef 2024-11-08T00:24:09,526 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/7965ad7c01134eb7bb15de5d414f08ef, entries=100, sequenceid=428, filesize=9.6 K 2024-11-08T00:24:09,528 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/.tmp/C/c413a16c45b249d7b4781e35b6a62253 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/c413a16c45b249d7b4781e35b6a62253 2024-11-08T00:24:09,549 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/c413a16c45b249d7b4781e35b6a62253, entries=100, sequenceid=428, filesize=9.6 K 2024-11-08T00:24:09,551 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for b211e3aadde87eeeb10e4f0dad149691 in 954ms, sequenceid=428, compaction requested=true 2024-11-08T00:24:09,556 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/91bbc5fe702443389e0f9734f8611b87, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/d22f82be612249ccb30f622750dec257, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/123a0cbb25724390bcbfe282f65a1354, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/d14b3c474e8b4937ab4416e97f37d803, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/d4985818e7d648d68004e55a13d5dfbb, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/a7368cc2155944a39cc48aabb0fd0c60, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/fdd0a475c7f44b5a88b4c51fdde08618, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/441d335b82014cd19a4de813ccc11a24, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/283a02a889d040e6bfe3c9950e72f3c1, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/f1769abfb5f04776884d7c9fb07e3d62, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/0b4348ac313144ba97de5e878c672caf, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/7fe36e514feb410ebfdcb1d24e56f443, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/b711aae2c9dd4aebb02abf5220fdc26b, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/ec5cb1fa8b114181a4bc33b2a64eaec6, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/7f912cb29b9c41b2a3d8949caf20ccc3, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/75e26036921541b28b5a6877e8436b65, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/8336736154244353aa7cc3d5d1f6bb6a, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/f4fbae6264f0415cb6ab90196ef06a4f, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/9d9199e43d974ad5bf77b42025a0bbb3, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/f581ed99fdc044d4b13364cdb424b26f, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/79502a1467ac426a919295bd5bfca51d, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/10be81670dbf48e980a91a3a3674afd2, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/cbc31695e2d043eeb657dd02d25f9edc, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/c6c0ae2b5533458f90ce9c4952199d45, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/523e55cf3e0c4c9fa876fcb6eedf1da7, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/e71ad53bea91438c9eb4a42a00e714e6, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/862766db44dd4a529a90184bfbff0751, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/862301e3db324648acf11363ae0daca7, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/ccb4e5978a9b4d6898b6e21f206b8cde] to archive 2024-11-08T00:24:09,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-08T00:24:09,562 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T00:24:09,575 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/91bbc5fe702443389e0f9734f8611b87 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/91bbc5fe702443389e0f9734f8611b87 2024-11-08T00:24:09,578 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/d22f82be612249ccb30f622750dec257 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/d22f82be612249ccb30f622750dec257 2024-11-08T00:24:09,580 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/123a0cbb25724390bcbfe282f65a1354 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/123a0cbb25724390bcbfe282f65a1354 2024-11-08T00:24:09,582 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/d14b3c474e8b4937ab4416e97f37d803 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/d14b3c474e8b4937ab4416e97f37d803 2024-11-08T00:24:09,585 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/d4985818e7d648d68004e55a13d5dfbb to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/d4985818e7d648d68004e55a13d5dfbb 2024-11-08T00:24:09,592 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/a7368cc2155944a39cc48aabb0fd0c60 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/a7368cc2155944a39cc48aabb0fd0c60 2024-11-08T00:24:09,600 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/fdd0a475c7f44b5a88b4c51fdde08618 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/fdd0a475c7f44b5a88b4c51fdde08618 2024-11-08T00:24:09,603 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/441d335b82014cd19a4de813ccc11a24 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/441d335b82014cd19a4de813ccc11a24 2024-11-08T00:24:09,605 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/283a02a889d040e6bfe3c9950e72f3c1 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/283a02a889d040e6bfe3c9950e72f3c1 2024-11-08T00:24:09,607 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/f1769abfb5f04776884d7c9fb07e3d62 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/f1769abfb5f04776884d7c9fb07e3d62 2024-11-08T00:24:09,609 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/0b4348ac313144ba97de5e878c672caf to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/0b4348ac313144ba97de5e878c672caf 2024-11-08T00:24:09,611 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/7fe36e514feb410ebfdcb1d24e56f443 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/7fe36e514feb410ebfdcb1d24e56f443 2024-11-08T00:24:09,614 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/b711aae2c9dd4aebb02abf5220fdc26b to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/b711aae2c9dd4aebb02abf5220fdc26b 2024-11-08T00:24:09,619 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/ec5cb1fa8b114181a4bc33b2a64eaec6 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/ec5cb1fa8b114181a4bc33b2a64eaec6 2024-11-08T00:24:09,621 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/7f912cb29b9c41b2a3d8949caf20ccc3 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/7f912cb29b9c41b2a3d8949caf20ccc3 2024-11-08T00:24:09,636 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/75e26036921541b28b5a6877e8436b65 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/75e26036921541b28b5a6877e8436b65 2024-11-08T00:24:09,638 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/8336736154244353aa7cc3d5d1f6bb6a to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/8336736154244353aa7cc3d5d1f6bb6a 2024-11-08T00:24:09,641 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/f4fbae6264f0415cb6ab90196ef06a4f to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/f4fbae6264f0415cb6ab90196ef06a4f 2024-11-08T00:24:09,643 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/9d9199e43d974ad5bf77b42025a0bbb3 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/9d9199e43d974ad5bf77b42025a0bbb3 2024-11-08T00:24:09,645 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/f581ed99fdc044d4b13364cdb424b26f to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/f581ed99fdc044d4b13364cdb424b26f 2024-11-08T00:24:09,647 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/79502a1467ac426a919295bd5bfca51d to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/79502a1467ac426a919295bd5bfca51d 2024-11-08T00:24:09,649 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/10be81670dbf48e980a91a3a3674afd2 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/10be81670dbf48e980a91a3a3674afd2 2024-11-08T00:24:09,650 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/cbc31695e2d043eeb657dd02d25f9edc to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/cbc31695e2d043eeb657dd02d25f9edc 2024-11-08T00:24:09,652 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/c6c0ae2b5533458f90ce9c4952199d45 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/c6c0ae2b5533458f90ce9c4952199d45 2024-11-08T00:24:09,654 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/523e55cf3e0c4c9fa876fcb6eedf1da7 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/523e55cf3e0c4c9fa876fcb6eedf1da7 2024-11-08T00:24:09,656 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/e71ad53bea91438c9eb4a42a00e714e6 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/e71ad53bea91438c9eb4a42a00e714e6 2024-11-08T00:24:09,658 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/862766db44dd4a529a90184bfbff0751 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/862766db44dd4a529a90184bfbff0751 2024-11-08T00:24:09,659 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/862301e3db324648acf11363ae0daca7 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/862301e3db324648acf11363ae0daca7 2024-11-08T00:24:09,661 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/ccb4e5978a9b4d6898b6e21f206b8cde to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/ccb4e5978a9b4d6898b6e21f206b8cde 2024-11-08T00:24:09,698 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/e9c38e2ce4fb48de8f8f3ca1d37be541, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/cff1bf3a5e50474d94bec51db2b9b811, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/960ca2b1bc6847eeba29b013dfa5d384, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/4844bc66e8924cacaff69f7993ddda06, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/2cd4bfaeb8294572a28fc37728698047, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/ab5437c759354644904545b32e364872, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/e2ae0a5be75d4e6ba65e0e36ada2966b, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/59a5b8b6c0a54bb2812effa120328eb1, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/f28c43f5ab8444f6a7b91f495359a5f1, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/3c5be6f01f4e49f69b05148f93e8799e, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/5209b6e937e2470faf7ecd725b34c000, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/9d8ca6499d774958846c2129dce08760, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/f00910855c134b9aaa363aaefdeb61e5, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/bae90d04dd544e399ced30caa2314b0d, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/190b95078c5b4a96bf6c1cb5387264c4, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/d3d53db3cd03469cbf5fdd8020f12b1d, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/944c0f6fe850433181571b1b153b8dc0, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/abce69cadf2e4d98a28107df8f370b3a, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/b41eb4738ebe42cf817856c8e774524d, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/d67f52b2905b4eeaacf851e06de189fe, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/cc9b14e5d465482fb8e1e708668296eb, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/179316b8bb1941eba33a8fd4d1e56978, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/ced4a99db97a4aa3836e244fddc57cd2, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/e3c73a74d66341a298325370d5c4e49f, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/06b49c5fda3b464aab0a26004d35296a, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/70c454d5a5a54433a1895abb09f4fd69, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/b6e699f6350d43f2a0014c796f80031d, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/dc2cff7f1b32484e8e78e7f5e9c0d8b4, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/cfa002d815224d159e82bea1157bab01] to archive 2024-11-08T00:24:09,705 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T00:24:09,721 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/e9c38e2ce4fb48de8f8f3ca1d37be541 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/e9c38e2ce4fb48de8f8f3ca1d37be541 2024-11-08T00:24:09,723 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/cff1bf3a5e50474d94bec51db2b9b811 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/cff1bf3a5e50474d94bec51db2b9b811 2024-11-08T00:24:09,726 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/960ca2b1bc6847eeba29b013dfa5d384 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/960ca2b1bc6847eeba29b013dfa5d384 2024-11-08T00:24:09,731 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/4844bc66e8924cacaff69f7993ddda06 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/4844bc66e8924cacaff69f7993ddda06 2024-11-08T00:24:09,738 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/2cd4bfaeb8294572a28fc37728698047 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/2cd4bfaeb8294572a28fc37728698047 2024-11-08T00:24:09,739 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/ab5437c759354644904545b32e364872 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/ab5437c759354644904545b32e364872 2024-11-08T00:24:09,756 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/e2ae0a5be75d4e6ba65e0e36ada2966b to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/e2ae0a5be75d4e6ba65e0e36ada2966b 2024-11-08T00:24:09,773 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/59a5b8b6c0a54bb2812effa120328eb1 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/59a5b8b6c0a54bb2812effa120328eb1 2024-11-08T00:24:09,782 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/f28c43f5ab8444f6a7b91f495359a5f1 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/f28c43f5ab8444f6a7b91f495359a5f1 2024-11-08T00:24:09,784 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/3c5be6f01f4e49f69b05148f93e8799e to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/3c5be6f01f4e49f69b05148f93e8799e 2024-11-08T00:24:09,802 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/5209b6e937e2470faf7ecd725b34c000 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/5209b6e937e2470faf7ecd725b34c000 2024-11-08T00:24:09,807 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/9d8ca6499d774958846c2129dce08760 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/9d8ca6499d774958846c2129dce08760 2024-11-08T00:24:09,811 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/f00910855c134b9aaa363aaefdeb61e5 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/f00910855c134b9aaa363aaefdeb61e5 2024-11-08T00:24:09,813 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/bae90d04dd544e399ced30caa2314b0d to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/bae90d04dd544e399ced30caa2314b0d 2024-11-08T00:24:09,817 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/190b95078c5b4a96bf6c1cb5387264c4 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/190b95078c5b4a96bf6c1cb5387264c4 2024-11-08T00:24:09,826 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/d3d53db3cd03469cbf5fdd8020f12b1d to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/d3d53db3cd03469cbf5fdd8020f12b1d 2024-11-08T00:24:09,828 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/944c0f6fe850433181571b1b153b8dc0 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/944c0f6fe850433181571b1b153b8dc0 2024-11-08T00:24:09,831 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/abce69cadf2e4d98a28107df8f370b3a to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/abce69cadf2e4d98a28107df8f370b3a 2024-11-08T00:24:09,832 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/b41eb4738ebe42cf817856c8e774524d to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/b41eb4738ebe42cf817856c8e774524d 2024-11-08T00:24:09,836 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/d67f52b2905b4eeaacf851e06de189fe to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/d67f52b2905b4eeaacf851e06de189fe 2024-11-08T00:24:09,838 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/cc9b14e5d465482fb8e1e708668296eb to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/cc9b14e5d465482fb8e1e708668296eb 2024-11-08T00:24:09,839 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/179316b8bb1941eba33a8fd4d1e56978 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/179316b8bb1941eba33a8fd4d1e56978 2024-11-08T00:24:09,844 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/ced4a99db97a4aa3836e244fddc57cd2 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/ced4a99db97a4aa3836e244fddc57cd2 2024-11-08T00:24:09,847 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/e3c73a74d66341a298325370d5c4e49f to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/e3c73a74d66341a298325370d5c4e49f 2024-11-08T00:24:09,856 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/06b49c5fda3b464aab0a26004d35296a to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/06b49c5fda3b464aab0a26004d35296a 2024-11-08T00:24:09,860 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/70c454d5a5a54433a1895abb09f4fd69 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/70c454d5a5a54433a1895abb09f4fd69 2024-11-08T00:24:09,868 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/b6e699f6350d43f2a0014c796f80031d to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/b6e699f6350d43f2a0014c796f80031d 2024-11-08T00:24:09,873 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/dc2cff7f1b32484e8e78e7f5e9c0d8b4 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/dc2cff7f1b32484e8e78e7f5e9c0d8b4 2024-11-08T00:24:09,875 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/cfa002d815224d159e82bea1157bab01 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/cfa002d815224d159e82bea1157bab01 2024-11-08T00:24:09,897 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/a76bf377a91f412f9b0bd2162c3efb82, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/cf2cbcadc64f4611b569592abf87a531, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/9ed79859e52e4defb670125cce956c03, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/29dd74f6fc1b42d7a7a1a07ff1056dfd, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/2cd8e63598b04501ad52d541451ee148, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/5ded9cb9e5d64f62ab7e0843cfb217b6, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/2fa338d8b64243efbf48b133750301fe, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/433ba9aae46847c3a2c3e661f61d8660, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/f5e4ace533af47d8969ae95395a87fa0, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/a82d8980c0ca4e598b8d282a6d79a4c0, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/6be675b73d3f498fbbe0008b0af0eaac, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/ac8b7963794349d395f49cda6ce4d286, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/5eca041cd9eb429ebea96c91267041ab, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/8160d7b06c634c75bb93314c779c9a89, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/e2ec732b48354270914a3382e9ba8339, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/857d720996364130b698bf37b1703e5f, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/d9f4bd1aa8f5441eace0e8319782c8a9, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/048757cf9b914a7caadb549dc0f7175f, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/1ee4468c62954d29935af3046f1380a9, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/6d5cd66872a24ce4a2519b451835a4d1, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/364aba4804234a38bd99d0336f5614c9, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/befee3af3a21497fa8d0d7698c47a7b7, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/00c229c532be4ceabb372b9045845c7e, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/3ab11fbb644e41e08007a66a894ac5de, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/00d8776ac832491d9317b9b01df042ce, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/ee1f46ff47c147b682905f87d466d2ff, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/6eeece1245d24f408819adfe600e3379, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/145c1ebe6c6b4dc2bddaf2473d28d87a, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/daeec0e384d742aea86bd17aea5e7697] to archive 2024-11-08T00:24:09,901 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T00:24:09,917 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/a76bf377a91f412f9b0bd2162c3efb82 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/a76bf377a91f412f9b0bd2162c3efb82 2024-11-08T00:24:09,928 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/cf2cbcadc64f4611b569592abf87a531 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/cf2cbcadc64f4611b569592abf87a531 2024-11-08T00:24:09,956 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/9ed79859e52e4defb670125cce956c03 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/9ed79859e52e4defb670125cce956c03 2024-11-08T00:24:09,961 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/29dd74f6fc1b42d7a7a1a07ff1056dfd to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/29dd74f6fc1b42d7a7a1a07ff1056dfd 2024-11-08T00:24:09,964 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/2cd8e63598b04501ad52d541451ee148 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/2cd8e63598b04501ad52d541451ee148 2024-11-08T00:24:09,966 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/5ded9cb9e5d64f62ab7e0843cfb217b6 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/5ded9cb9e5d64f62ab7e0843cfb217b6 2024-11-08T00:24:09,969 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/2fa338d8b64243efbf48b133750301fe to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/2fa338d8b64243efbf48b133750301fe 2024-11-08T00:24:09,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/433ba9aae46847c3a2c3e661f61d8660 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/433ba9aae46847c3a2c3e661f61d8660 2024-11-08T00:24:09,973 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/f5e4ace533af47d8969ae95395a87fa0 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/f5e4ace533af47d8969ae95395a87fa0 2024-11-08T00:24:09,974 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/a82d8980c0ca4e598b8d282a6d79a4c0 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/a82d8980c0ca4e598b8d282a6d79a4c0 2024-11-08T00:24:09,976 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/6be675b73d3f498fbbe0008b0af0eaac to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/6be675b73d3f498fbbe0008b0af0eaac 2024-11-08T00:24:09,981 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/ac8b7963794349d395f49cda6ce4d286 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/ac8b7963794349d395f49cda6ce4d286 2024-11-08T00:24:09,983 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/5eca041cd9eb429ebea96c91267041ab to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/5eca041cd9eb429ebea96c91267041ab 2024-11-08T00:24:09,985 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/8160d7b06c634c75bb93314c779c9a89 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/8160d7b06c634c75bb93314c779c9a89 2024-11-08T00:24:09,987 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/e2ec732b48354270914a3382e9ba8339 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/e2ec732b48354270914a3382e9ba8339 2024-11-08T00:24:09,989 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/857d720996364130b698bf37b1703e5f to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/857d720996364130b698bf37b1703e5f 2024-11-08T00:24:09,991 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/d9f4bd1aa8f5441eace0e8319782c8a9 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/d9f4bd1aa8f5441eace0e8319782c8a9 2024-11-08T00:24:09,996 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/048757cf9b914a7caadb549dc0f7175f to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/048757cf9b914a7caadb549dc0f7175f 2024-11-08T00:24:09,998 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/1ee4468c62954d29935af3046f1380a9 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/1ee4468c62954d29935af3046f1380a9 2024-11-08T00:24:10,002 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/6d5cd66872a24ce4a2519b451835a4d1 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/6d5cd66872a24ce4a2519b451835a4d1 2024-11-08T00:24:10,004 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/364aba4804234a38bd99d0336f5614c9 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/364aba4804234a38bd99d0336f5614c9 2024-11-08T00:24:10,007 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/befee3af3a21497fa8d0d7698c47a7b7 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/befee3af3a21497fa8d0d7698c47a7b7 2024-11-08T00:24:10,009 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/00c229c532be4ceabb372b9045845c7e to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/00c229c532be4ceabb372b9045845c7e 2024-11-08T00:24:10,011 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/3ab11fbb644e41e08007a66a894ac5de to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/3ab11fbb644e41e08007a66a894ac5de 2024-11-08T00:24:10,014 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/00d8776ac832491d9317b9b01df042ce to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/00d8776ac832491d9317b9b01df042ce 2024-11-08T00:24:10,016 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/ee1f46ff47c147b682905f87d466d2ff to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/ee1f46ff47c147b682905f87d466d2ff 2024-11-08T00:24:10,018 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/6eeece1245d24f408819adfe600e3379 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/6eeece1245d24f408819adfe600e3379 2024-11-08T00:24:10,020 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/145c1ebe6c6b4dc2bddaf2473d28d87a to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/145c1ebe6c6b4dc2bddaf2473d28d87a 2024-11-08T00:24:10,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/daeec0e384d742aea86bd17aea5e7697 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/daeec0e384d742aea86bd17aea5e7697 2024-11-08T00:24:10,042 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/recovered.edits/431.seqid, newMaxSeqId=431, maxSeqId=1 2024-11-08T00:24:10,043 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. 2024-11-08T00:24:10,043 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1676): Region close journal for b211e3aadde87eeeb10e4f0dad149691: Waiting for close lock at 1731025448597Running coprocessor pre-close hooks at 1731025448597Disabling compacts and flushes for region at 1731025448597Disabling writes for close at 1731025448597Obtaining lock to block concurrent updates at 1731025448597Preparing flush snapshotting stores in b211e3aadde87eeeb10e4f0dad149691 at 1731025448597Finished memstore snapshotting TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1731025448598 (+1 ms)Flushing stores of TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691. at 1731025448598Flushing b211e3aadde87eeeb10e4f0dad149691/A: creating writer at 1731025448599 (+1 ms)Flushing b211e3aadde87eeeb10e4f0dad149691/A: appending metadata at 1731025448602 (+3 ms)Flushing b211e3aadde87eeeb10e4f0dad149691/A: closing flushed file at 1731025448602Flushing b211e3aadde87eeeb10e4f0dad149691/B: creating writer at 1731025449021 (+419 ms)Flushing b211e3aadde87eeeb10e4f0dad149691/B: appending metadata at 1731025449025 (+4 ms)Flushing b211e3aadde87eeeb10e4f0dad149691/B: closing flushed file at 1731025449025Flushing b211e3aadde87eeeb10e4f0dad149691/C: creating writer at 1731025449449 (+424 ms)Flushing b211e3aadde87eeeb10e4f0dad149691/C: appending metadata at 1731025449470 (+21 ms)Flushing b211e3aadde87eeeb10e4f0dad149691/C: closing flushed file at 1731025449470Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5581c522: reopening flushed file at 1731025449501 (+31 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@78d49398: reopening flushed file at 1731025449511 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5008858c: reopening flushed file at 1731025449526 (+15 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for b211e3aadde87eeeb10e4f0dad149691 in 954ms, sequenceid=428, compaction requested=true at 1731025449551 (+25 ms)Writing region close event to WAL at 1731025450030 (+479 ms)Running coprocessor post-close hooks at 1731025450043 (+13 ms)Closed at 1731025450043 2024-11-08T00:24:10,048 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=b211e3aadde87eeeb10e4f0dad149691, regionState=CLOSED 2024-11-08T00:24:10,051 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 because future has completed 2024-11-08T00:24:10,049 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(157): Closed b211e3aadde87eeeb10e4f0dad149691 2024-11-08T00:24:10,068 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-08T00:24:10,068 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseRegionProcedure b211e3aadde87eeeb10e4f0dad149691, server=3302f0f507bd,40479,1731025415856 in 1.6330 sec 2024-11-08T00:24:10,070 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-08T00:24:10,070 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b211e3aadde87eeeb10e4f0dad149691, UNASSIGN in 1.6560 sec 2024-11-08T00:24:10,074 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-08T00:24:10,074 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6680 sec 2024-11-08T00:24:10,077 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731025450077"}]},"ts":"1731025450077"} 2024-11-08T00:24:10,079 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-08T00:24:10,079 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-08T00:24:10,083 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6920 sec 2024-11-08T00:24:10,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-08T00:24:10,572 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:24:10,576 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-08T00:24:10,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T00:24:10,584 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T00:24:10,586 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=31, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T00:24:10,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-08T00:24:10,612 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691 2024-11-08T00:24:10,641 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A, FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B, FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C, FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/recovered.edits] 2024-11-08T00:24:10,673 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/1bef880d258b44b1ab7047fa99e26d2d to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/1bef880d258b44b1ab7047fa99e26d2d 2024-11-08T00:24:10,683 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/b8b7501df5cd4484b8cda7c57bb27e67 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/b8b7501df5cd4484b8cda7c57bb27e67 2024-11-08T00:24:10,686 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/f080dc006e0f4be7af98f941e88d7289 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/A/f080dc006e0f4be7af98f941e88d7289 2024-11-08T00:24:10,690 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/256950ef1fa94fabae806de30ceed13a to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/256950ef1fa94fabae806de30ceed13a 2024-11-08T00:24:10,692 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/3c6e0fcff4234ba3bfbe87a1b1164a5f to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/3c6e0fcff4234ba3bfbe87a1b1164a5f 2024-11-08T00:24:10,694 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/7965ad7c01134eb7bb15de5d414f08ef to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/B/7965ad7c01134eb7bb15de5d414f08ef 2024-11-08T00:24:10,700 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/50182074d73a4cd997c2e58eb4b31cdb to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/50182074d73a4cd997c2e58eb4b31cdb 2024-11-08T00:24:10,703 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/c413a16c45b249d7b4781e35b6a62253 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/c413a16c45b249d7b4781e35b6a62253 2024-11-08T00:24:10,705 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/ebecb70781594987a96c65469ee5379c to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/C/ebecb70781594987a96c65469ee5379c 2024-11-08T00:24:10,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-08T00:24:10,711 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/recovered.edits/431.seqid to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691/recovered.edits/431.seqid 2024-11-08T00:24:10,713 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/b211e3aadde87eeeb10e4f0dad149691 2024-11-08T00:24:10,713 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-08T00:24:10,720 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=31, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T00:24:10,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-08T00:24:10,758 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-08T00:24:10,856 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-08T00:24:10,868 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=31, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T00:24:10,868 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-08T00:24:10,869 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731025450868"}]},"ts":"9223372036854775807"} 2024-11-08T00:24:10,875 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-08T00:24:10,875 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => b211e3aadde87eeeb10e4f0dad149691, NAME => 'TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691.', STARTKEY => '', ENDKEY => ''}] 2024-11-08T00:24:10,875 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-08T00:24:10,875 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731025450875"}]},"ts":"9223372036854775807"} 2024-11-08T00:24:10,882 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-08T00:24:10,885 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=31, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T00:24:10,888 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 308 msec 2024-11-08T00:24:10,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-08T00:24:10,919 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-08T00:24:10,919 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:24:10,957 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=237 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1477324790_22 at /127.0.0.1:38740 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;3302f0f507bd:40479-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=682 (was 598) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4474 (was 5092) 2024-11-08T00:24:11,000 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=237, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=682, ProcessCount=11, AvailableMemoryMB=4469 2024-11-08T00:24:11,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-08T00:24:11,005 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T00:24:11,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-08T00:24:11,009 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T00:24:11,009 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:11,010 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T00:24:11,016 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 32 2024-11-08T00:24:11,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-08T00:24:11,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741933_1109 (size=959) 2024-11-08T00:24:11,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-08T00:24:11,133 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f 2024-11-08T00:24:11,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741934_1110 (size=53) 2024-11-08T00:24:11,184 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:24:11,184 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing d9a7fdf8b82889b95543ffed36720442, disabling compactions & flushes 2024-11-08T00:24:11,184 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:11,185 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:11,185 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. after waiting 0 ms 2024-11-08T00:24:11,185 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:11,185 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:11,185 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for d9a7fdf8b82889b95543ffed36720442: Waiting for close lock at 1731025451184Disabling compacts and flushes for region at 1731025451184Disabling writes for close at 1731025451185 (+1 ms)Writing region close event to WAL at 1731025451185Closed at 1731025451185 2024-11-08T00:24:11,187 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T00:24:11,187 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731025451187"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731025451187"}]},"ts":"1731025451187"} 2024-11-08T00:24:11,191 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-08T00:24:11,193 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T00:24:11,193 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731025451193"}]},"ts":"1731025451193"} 2024-11-08T00:24:11,204 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-08T00:24:11,205 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d9a7fdf8b82889b95543ffed36720442, ASSIGN}] 2024-11-08T00:24:11,206 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d9a7fdf8b82889b95543ffed36720442, ASSIGN 2024-11-08T00:24:11,207 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d9a7fdf8b82889b95543ffed36720442, ASSIGN; state=OFFLINE, location=3302f0f507bd,40479,1731025415856; forceNewPlan=false, retain=false 2024-11-08T00:24:11,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-08T00:24:11,358 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=d9a7fdf8b82889b95543ffed36720442, regionState=OPENING, regionLocation=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:11,366 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d9a7fdf8b82889b95543ffed36720442, ASSIGN because future has completed 2024-11-08T00:24:11,367 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856}] 2024-11-08T00:24:11,524 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:11,524 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} 2024-11-08T00:24:11,525 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:11,525 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:24:11,525 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:11,525 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:11,527 INFO [StoreOpener-d9a7fdf8b82889b95543ffed36720442-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:11,534 INFO [StoreOpener-d9a7fdf8b82889b95543ffed36720442-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d9a7fdf8b82889b95543ffed36720442 columnFamilyName A 2024-11-08T00:24:11,534 DEBUG [StoreOpener-d9a7fdf8b82889b95543ffed36720442-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:11,537 INFO [StoreOpener-d9a7fdf8b82889b95543ffed36720442-1 {}] regionserver.HStore(327): Store=d9a7fdf8b82889b95543ffed36720442/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:24:11,537 INFO [StoreOpener-d9a7fdf8b82889b95543ffed36720442-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:11,540 INFO [StoreOpener-d9a7fdf8b82889b95543ffed36720442-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d9a7fdf8b82889b95543ffed36720442 columnFamilyName B 2024-11-08T00:24:11,540 DEBUG [StoreOpener-d9a7fdf8b82889b95543ffed36720442-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:11,541 INFO [StoreOpener-d9a7fdf8b82889b95543ffed36720442-1 {}] regionserver.HStore(327): Store=d9a7fdf8b82889b95543ffed36720442/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:24:11,541 INFO [StoreOpener-d9a7fdf8b82889b95543ffed36720442-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:11,543 INFO [StoreOpener-d9a7fdf8b82889b95543ffed36720442-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d9a7fdf8b82889b95543ffed36720442 columnFamilyName C 2024-11-08T00:24:11,543 DEBUG [StoreOpener-d9a7fdf8b82889b95543ffed36720442-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:11,544 INFO [StoreOpener-d9a7fdf8b82889b95543ffed36720442-1 {}] regionserver.HStore(327): Store=d9a7fdf8b82889b95543ffed36720442/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:24:11,544 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:11,546 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:11,546 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:11,548 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:11,548 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:11,549 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-08T00:24:11,552 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:11,561 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T00:24:11,562 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened d9a7fdf8b82889b95543ffed36720442; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70212276, jitterRate=0.04624444246292114}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-08T00:24:11,562 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:11,563 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for d9a7fdf8b82889b95543ffed36720442: Running coprocessor pre-open hook at 1731025451525Writing region info on filesystem at 1731025451525Initializing all the Stores at 1731025451527 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731025451527Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731025451527Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731025451527Cleaning up temporary data from old regions at 1731025451548 (+21 ms)Running coprocessor post-open hooks at 1731025451562 (+14 ms)Region opened successfully at 1731025451563 (+1 ms) 2024-11-08T00:24:11,564 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., pid=34, masterSystemTime=1731025451519 2024-11-08T00:24:11,569 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=d9a7fdf8b82889b95543ffed36720442, regionState=OPEN, openSeqNum=2, regionLocation=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:11,571 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 because future has completed 2024-11-08T00:24:11,573 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:11,573 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:11,585 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-08T00:24:11,585 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 in 216 msec 2024-11-08T00:24:11,590 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T00:24:11,591 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731025451590"}]},"ts":"1731025451590"} 2024-11-08T00:24:11,592 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-08T00:24:11,592 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d9a7fdf8b82889b95543ffed36720442, ASSIGN in 380 msec 2024-11-08T00:24:11,594 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-08T00:24:11,596 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T00:24:11,600 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 591 msec 2024-11-08T00:24:11,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-08T00:24:11,652 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:24:11,665 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@758e8028, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:24:11,665 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:24:11,665 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:24:11,673 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:24:11,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:24:11,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:24:11,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bc11423, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:24:11,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:24:11,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:24:11,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:24:11,679 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56812, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:24:11,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f1c755, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:24:11,681 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:24:11,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:24:11,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:24:11,687 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49966, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:24:11,689 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-08T00:24:11,691 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 3302f0f507bd,36785,1731025414356 2024-11-08T00:24:11,691 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6ad5e6af 2024-11-08T00:24:11,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-08T00:24:11,693 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56826, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-08T00:24:11,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-08T00:24:11,718 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T00:24:11,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-08T00:24:11,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741935_1111 (size=995) 2024-11-08T00:24:11,811 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-08T00:24:11,812 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-08T00:24:11,823 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-08T00:24:11,841 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d9a7fdf8b82889b95543ffed36720442, REOPEN/MOVE}] 2024-11-08T00:24:11,847 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d9a7fdf8b82889b95543ffed36720442, REOPEN/MOVE 2024-11-08T00:24:11,848 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=d9a7fdf8b82889b95543ffed36720442, regionState=CLOSING, regionLocation=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:11,856 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d9a7fdf8b82889b95543ffed36720442, REOPEN/MOVE because future has completed 2024-11-08T00:24:11,858 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-08T00:24:11,858 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856}] 2024-11-08T00:24:12,011 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(122): Close d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:12,012 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-08T00:24:12,012 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1722): Closing d9a7fdf8b82889b95543ffed36720442, disabling compactions & flushes 2024-11-08T00:24:12,012 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:12,012 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:12,012 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. after waiting 0 ms 2024-11-08T00:24:12,012 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:12,036 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T00:24:12,037 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:12,037 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1676): Region close journal for d9a7fdf8b82889b95543ffed36720442: Waiting for close lock at 1731025452012Running coprocessor pre-close hooks at 1731025452012Disabling compacts and flushes for region at 1731025452012Disabling writes for close at 1731025452012Writing region close event to WAL at 1731025452016 (+4 ms)Running coprocessor post-close hooks at 1731025452037 (+21 ms)Closed at 1731025452037 2024-11-08T00:24:12,037 WARN [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegionServer(3299): Not adding moved region record: d9a7fdf8b82889b95543ffed36720442 to self. 2024-11-08T00:24:12,040 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(157): Closed d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:12,042 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=d9a7fdf8b82889b95543ffed36720442, regionState=CLOSED 2024-11-08T00:24:12,049 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 because future has completed 2024-11-08T00:24:12,063 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-08T00:24:12,063 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; CloseRegionProcedure d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 in 203 msec 2024-11-08T00:24:12,066 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d9a7fdf8b82889b95543ffed36720442, REOPEN/MOVE; state=CLOSED, location=3302f0f507bd,40479,1731025415856; forceNewPlan=false, retain=true 2024-11-08T00:24:12,218 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=d9a7fdf8b82889b95543ffed36720442, regionState=OPENING, regionLocation=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:12,221 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d9a7fdf8b82889b95543ffed36720442, REOPEN/MOVE because future has completed 2024-11-08T00:24:12,221 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856}] 2024-11-08T00:24:12,379 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:12,379 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} 2024-11-08T00:24:12,380 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:12,380 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:24:12,380 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:12,380 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:12,384 INFO [StoreOpener-d9a7fdf8b82889b95543ffed36720442-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:12,394 INFO [StoreOpener-d9a7fdf8b82889b95543ffed36720442-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d9a7fdf8b82889b95543ffed36720442 columnFamilyName A 2024-11-08T00:24:12,401 DEBUG [StoreOpener-d9a7fdf8b82889b95543ffed36720442-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:12,401 INFO [StoreOpener-d9a7fdf8b82889b95543ffed36720442-1 {}] regionserver.HStore(327): Store=d9a7fdf8b82889b95543ffed36720442/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:24:12,402 INFO [StoreOpener-d9a7fdf8b82889b95543ffed36720442-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:12,403 INFO [StoreOpener-d9a7fdf8b82889b95543ffed36720442-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d9a7fdf8b82889b95543ffed36720442 columnFamilyName B 2024-11-08T00:24:12,403 DEBUG [StoreOpener-d9a7fdf8b82889b95543ffed36720442-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:12,404 INFO [StoreOpener-d9a7fdf8b82889b95543ffed36720442-1 {}] regionserver.HStore(327): Store=d9a7fdf8b82889b95543ffed36720442/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:24:12,404 INFO [StoreOpener-d9a7fdf8b82889b95543ffed36720442-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:12,405 INFO [StoreOpener-d9a7fdf8b82889b95543ffed36720442-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d9a7fdf8b82889b95543ffed36720442 columnFamilyName C 2024-11-08T00:24:12,405 DEBUG [StoreOpener-d9a7fdf8b82889b95543ffed36720442-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:12,406 INFO [StoreOpener-d9a7fdf8b82889b95543ffed36720442-1 {}] regionserver.HStore(327): Store=d9a7fdf8b82889b95543ffed36720442/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:24:12,406 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:12,409 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:12,411 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:12,424 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:12,424 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:12,425 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-08T00:24:12,426 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:12,427 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened d9a7fdf8b82889b95543ffed36720442; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66994214, jitterRate=-0.0017084181308746338}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-08T00:24:12,428 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:12,429 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for d9a7fdf8b82889b95543ffed36720442: Running coprocessor pre-open hook at 1731025452380Writing region info on filesystem at 1731025452380Initializing all the Stores at 1731025452383 (+3 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731025452383Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731025452383Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731025452383Cleaning up temporary data from old regions at 1731025452424 (+41 ms)Running coprocessor post-open hooks at 1731025452428 (+4 ms)Region opened successfully at 1731025452429 (+1 ms) 2024-11-08T00:24:12,430 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., pid=39, masterSystemTime=1731025452373 2024-11-08T00:24:12,433 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:12,433 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:12,433 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=d9a7fdf8b82889b95543ffed36720442, regionState=OPEN, openSeqNum=5, regionLocation=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:12,435 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 because future has completed 2024-11-08T00:24:12,440 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=37 2024-11-08T00:24:12,441 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 in 215 msec 2024-11-08T00:24:12,446 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-08T00:24:12,446 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d9a7fdf8b82889b95543ffed36720442, REOPEN/MOVE in 600 msec 2024-11-08T00:24:12,454 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-08T00:24:12,454 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 626 msec 2024-11-08T00:24:12,469 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 741 msec 2024-11-08T00:24:12,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-08T00:24:12,476 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:24:12,480 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24808cf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:24:12,480 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:24:12,481 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:24:12,482 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:24:12,483 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:24:12,483 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:24:12,483 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d8bf32, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:24:12,483 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:24:12,483 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:24:12,484 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:24:12,486 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44454, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:24:12,487 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2997a437, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:24:12,496 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1edc08a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:24:12,496 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:24:12,497 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:24:12,499 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:24:12,499 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:24:12,499 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:24:12,500 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62990a51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:24:12,500 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:24:12,500 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:24:12,500 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:24:12,502 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44474, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:24:12,503 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71b2717, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:24:12,506 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d7f1da0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:24:12,506 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:24:12,506 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:24:12,507 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:24:12,508 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:24:12,508 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:24:12,508 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@767b83f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:24:12,508 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:24:12,508 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:24:12,509 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:24:12,510 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44484, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:24:12,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@677ae992, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:24:12,513 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63f4ae4b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:24:12,513 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:24:12,513 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:24:12,514 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:24:12,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:24:12,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:24:12,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f53dd0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:24:12,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:24:12,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:24:12,516 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:24:12,517 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44504, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:24:12,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25213d7e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:24:12,518 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35da843c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:24:12,518 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:24:12,518 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:24:12,520 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:24:12,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:24:12,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:24:12,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@312b3193, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:24:12,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:24:12,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:24:12,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:24:12,533 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44522, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:24:12,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54e25876, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:24:12,536 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a1653a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:24:12,536 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:24:12,536 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:24:12,538 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:24:12,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:24:12,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:24:12,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@798a2944, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:24:12,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:24:12,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:24:12,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:24:12,541 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44538, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:24:12,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ff1fb3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:24:12,544 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40ce1eea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:24:12,544 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:24:12,544 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:24:12,545 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:24:12,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:24:12,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:24:12,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6199ecd9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:24:12,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:24:12,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:24:12,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:24:12,549 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44542, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:24:12,552 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@621343d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:24:12,556 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16bb65dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:24:12,557 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:24:12,557 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:24:12,558 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:24:12,558 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:24:12,558 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:24:12,559 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79aee5b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:24:12,559 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:24:12,559 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:24:12,560 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:24:12,562 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44554, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:24:12,563 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@596805ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:24:12,564 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70861d0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:24:12,564 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:24:12,565 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:24:12,566 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:24:12,566 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:24:12,566 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:24:12,567 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25de4b30, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:24:12,567 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:24:12,567 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:24:12,568 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:24:12,569 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44562, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:24:12,570 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9778420, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:24:12,577 DEBUG [Thread-580 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:24:12,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:24:12,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:24:12,581 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57288, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:24:12,584 DEBUG [Thread-576 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:24:12,587 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5] 2024-11-08T00:24:12,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:24:12,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:24:12,594 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57290, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:24:12,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5] 2024-11-08T00:24:12,601 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-08T00:24:12,602 DEBUG [Thread-589 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:24:12,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-08T00:24:12,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-08T00:24:12,606 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:24:12,606 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:24:12,607 DEBUG [Thread-582 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:24:12,609 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57306, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:24:12,610 DEBUG [Thread-578 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:24:12,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:24:12,611 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T00:24:12,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:24:12,612 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5] 2024-11-08T00:24:12,613 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57320, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:24:12,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:24:12,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:24:12,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5] 2024-11-08T00:24:12,617 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T00:24:12,617 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T00:24:12,618 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57324, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:24:12,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5] 2024-11-08T00:24:12,634 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:24:12,634 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d9a7fdf8b82889b95543ffed36720442 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-08T00:24:12,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:12,672 DEBUG [Thread-591 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:24:12,673 DEBUG [Thread-591 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:24:12,674 DEBUG [Thread-591 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:24:12,676 DEBUG [Thread-574 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:24:12,677 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57330, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:24:12,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:12,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:12,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57290 deadline: 1731025512678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:12,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5] 2024-11-08T00:24:12,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:12,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:12,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:12,685 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:24:12,686 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:24:12,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:12,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57320 deadline: 1731025512684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:12,687 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57332, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:24:12,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:12,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:12,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:12,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025512685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:12,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:12,688 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:12,688 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:12,688 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:12,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5] 2024-11-08T00:24:12,696 DEBUG [Thread-585 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:24:12,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:24:12,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:24:12,700 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57338, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:24:12,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5] 2024-11-08T00:24:12,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57324 deadline: 1731025512675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:12,709 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:12,709 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:12,709 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:12,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:12,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 1 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025512690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:12,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:12,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:12,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:12,724 DEBUG [Thread-587 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:24:12,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-08T00:24:12,725 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:24:12,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:24:12,727 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57340, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:24:12,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5] 2024-11-08T00:24:12,761 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108e8b210ec99ca4b2397755414ebd07bae_d9a7fdf8b82889b95543ffed36720442 is 50, key is test_row_0/A:col10/1731025452633/Put/seqid=0 2024-11-08T00:24:12,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-08T00:24:12,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:12,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. as already flushing 2024-11-08T00:24:12,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:12,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:12,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:12,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:12,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:12,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57290 deadline: 1731025512800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:12,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:12,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025512805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:12,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:12,841 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:12,841 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:12,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57320 deadline: 1731025512805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:12,841 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:12,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:12,842 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:12,842 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:12,842 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:12,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 2 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025512821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:12,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:12,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:12,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:12,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:12,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:12,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:12,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741936_1112 (size=19474) 2024-11-08T00:24:12,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:12,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57324 deadline: 1731025512843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:12,851 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:12,851 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:12,851 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:12,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-08T00:24:12,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-08T00:24:12,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:12,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. as already flushing 2024-11-08T00:24:12,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:12,940 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:12,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:12,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:13,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:13,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:13,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57320 deadline: 1731025513049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:13,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:13,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:13,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:13,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:13,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57290 deadline: 1731025513049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:13,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:13,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:13,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:13,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:13,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025513049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:13,074 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:13,074 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:13,074 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:13,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025513048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:13,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:13,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57324 deadline: 1731025513057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:13,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:13,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:13,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:13,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:13,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:13,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:13,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-08T00:24:13,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:13,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. as already flushing 2024-11-08T00:24:13,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:13,096 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:13,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:13,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:13,247 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:13,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-08T00:24:13,252 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-08T00:24:13,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:13,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. as already flushing 2024-11-08T00:24:13,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:13,253 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:13,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:13,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:13,275 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108e8b210ec99ca4b2397755414ebd07bae_d9a7fdf8b82889b95543ffed36720442 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108e8b210ec99ca4b2397755414ebd07bae_d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:13,281 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/ce2b1e6d4cf84bd3803b3aa60b6b6a7b, store: [table=TestAcidGuarantees family=A region=d9a7fdf8b82889b95543ffed36720442] 2024-11-08T00:24:13,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/ce2b1e6d4cf84bd3803b3aa60b6b6a7b is 175, key is test_row_0/A:col10/1731025452633/Put/seqid=0 2024-11-08T00:24:13,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741937_1113 (size=56733) 2024-11-08T00:24:13,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:13,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57320 deadline: 1731025513377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:13,385 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:13,385 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:13,385 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:13,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:13,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57290 deadline: 1731025513378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:13,388 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:13,388 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:13,388 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:13,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:13,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025513388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:13,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:13,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:13,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:13,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:13,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025513389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:13,392 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:13,392 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:13,393 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:13,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:13,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57324 deadline: 1731025513389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:13,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:13,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:13,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:13,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-08T00:24:13,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:13,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. as already flushing 2024-11-08T00:24:13,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:13,412 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:13,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:13,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:13,571 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-08T00:24:13,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:13,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. as already flushing 2024-11-08T00:24:13,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:13,572 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:13,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:13,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:13,728 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-08T00:24:13,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:13,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. as already flushing 2024-11-08T00:24:13,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:13,728 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:13,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:13,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:13,756 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=19, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/ce2b1e6d4cf84bd3803b3aa60b6b6a7b 2024-11-08T00:24:13,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-08T00:24:13,845 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/5089aa09a49243f98b336765b4ff6045 is 50, key is test_row_0/B:col10/1731025452633/Put/seqid=0 2024-11-08T00:24:13,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741938_1114 (size=12001) 2024-11-08T00:24:13,871 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=19 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/5089aa09a49243f98b336765b4ff6045 2024-11-08T00:24:13,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-08T00:24:13,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:13,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. as already flushing 2024-11-08T00:24:13,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:13,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:13,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:13,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:13,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:13,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57320 deadline: 1731025513897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:13,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:13,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:13,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:13,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:13,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57290 deadline: 1731025513898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:13,906 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:13,906 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:13,906 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:13,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:13,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:13,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025513898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:13,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025513899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:13,907 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:13,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:13,907 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:13,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:13,907 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:13,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:13,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:13,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57324 deadline: 1731025513907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:13,909 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:13,909 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:13,909 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:13,948 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/644f453fa6b14d8baf64468e5ca7c77e is 50, key is test_row_0/C:col10/1731025452633/Put/seqid=0 2024-11-08T00:24:13,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741939_1115 (size=12001) 2024-11-08T00:24:13,996 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=19 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/644f453fa6b14d8baf64468e5ca7c77e 2024-11-08T00:24:14,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/ce2b1e6d4cf84bd3803b3aa60b6b6a7b as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/ce2b1e6d4cf84bd3803b3aa60b6b6a7b 2024-11-08T00:24:14,011 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/ce2b1e6d4cf84bd3803b3aa60b6b6a7b, entries=300, sequenceid=19, filesize=55.4 K 2024-11-08T00:24:14,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/5089aa09a49243f98b336765b4ff6045 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/5089aa09a49243f98b336765b4ff6045 2024-11-08T00:24:14,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-08T00:24:14,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:14,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. as already flushing 2024-11-08T00:24:14,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:14,046 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:14,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:14,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:14,059 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/5089aa09a49243f98b336765b4ff6045, entries=150, sequenceid=19, filesize=11.7 K 2024-11-08T00:24:14,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/644f453fa6b14d8baf64468e5ca7c77e as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/644f453fa6b14d8baf64468e5ca7c77e 2024-11-08T00:24:14,079 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/644f453fa6b14d8baf64468e5ca7c77e, entries=150, sequenceid=19, filesize=11.7 K 2024-11-08T00:24:14,093 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for d9a7fdf8b82889b95543ffed36720442 in 1458ms, sequenceid=19, compaction requested=false 2024-11-08T00:24:14,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d9a7fdf8b82889b95543ffed36720442: 2024-11-08T00:24:14,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-08T00:24:14,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:14,201 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing d9a7fdf8b82889b95543ffed36720442 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-08T00:24:14,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108d5b7854c750a47e188c680ba6c83f188_d9a7fdf8b82889b95543ffed36720442 is 50, key is test_row_0/A:col10/1731025452663/Put/seqid=0 2024-11-08T00:24:14,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741940_1116 (size=12154) 2024-11-08T00:24:14,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:14,324 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108d5b7854c750a47e188c680ba6c83f188_d9a7fdf8b82889b95543ffed36720442 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108d5b7854c750a47e188c680ba6c83f188_d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:14,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/2e23c1ee94804405839ddcab4fe4b70b, store: [table=TestAcidGuarantees family=A region=d9a7fdf8b82889b95543ffed36720442] 2024-11-08T00:24:14,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/2e23c1ee94804405839ddcab4fe4b70b is 175, key is test_row_0/A:col10/1731025452663/Put/seqid=0 2024-11-08T00:24:14,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741941_1117 (size=30955) 2024-11-08T00:24:14,376 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/2e23c1ee94804405839ddcab4fe4b70b 2024-11-08T00:24:14,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/6a46e44a3c624f06ae4619b5306e28b1 is 50, key is test_row_0/B:col10/1731025452663/Put/seqid=0 2024-11-08T00:24:14,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741942_1118 (size=12001) 2024-11-08T00:24:14,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-08T00:24:14,858 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/6a46e44a3c624f06ae4619b5306e28b1 2024-11-08T00:24:14,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/22d123ca509a48b4ae2398eda8759ae3 is 50, key is test_row_0/C:col10/1731025452663/Put/seqid=0 2024-11-08T00:24:14,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. as already flushing 2024-11-08T00:24:14,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:14,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741943_1119 (size=12001) 2024-11-08T00:24:14,977 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/22d123ca509a48b4ae2398eda8759ae3 2024-11-08T00:24:14,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:14,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57320 deadline: 1731025514962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:14,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:14,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57324 deadline: 1731025514967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:14,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:14,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:14,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:14,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:14,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57290 deadline: 1731025514977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:14,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:14,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:14,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:14,983 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:14,983 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:14,983 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:14,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:14,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025514969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:14,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:14,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:14,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:14,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:14,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025514983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:14,990 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:14,990 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:14,990 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:15,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/2e23c1ee94804405839ddcab4fe4b70b as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/2e23c1ee94804405839ddcab4fe4b70b 2024-11-08T00:24:15,030 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/2e23c1ee94804405839ddcab4fe4b70b, entries=150, sequenceid=41, filesize=30.2 K 2024-11-08T00:24:15,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/6a46e44a3c624f06ae4619b5306e28b1 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/6a46e44a3c624f06ae4619b5306e28b1 2024-11-08T00:24:15,054 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/6a46e44a3c624f06ae4619b5306e28b1, entries=150, sequenceid=41, filesize=11.7 K 2024-11-08T00:24:15,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/22d123ca509a48b4ae2398eda8759ae3 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/22d123ca509a48b4ae2398eda8759ae3 2024-11-08T00:24:15,071 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/22d123ca509a48b4ae2398eda8759ae3, entries=150, sequenceid=41, filesize=11.7 K 2024-11-08T00:24:15,073 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for d9a7fdf8b82889b95543ffed36720442 in 871ms, sequenceid=41, compaction requested=false 2024-11-08T00:24:15,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for d9a7fdf8b82889b95543ffed36720442: 2024-11-08T00:24:15,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:15,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-08T00:24:15,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-08T00:24:15,083 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-08T00:24:15,083 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4580 sec 2024-11-08T00:24:15,090 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 2.4830 sec 2024-11-08T00:24:15,126 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:24:15,126 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d9a7fdf8b82889b95543ffed36720442 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-08T00:24:15,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:15,177 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108c02b7b5ec1004224bcdc2bbc1501ad71_d9a7fdf8b82889b95543ffed36720442 is 50, key is test_row_0/A:col10/1731025454978/Put/seqid=0 2024-11-08T00:24:15,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:15,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57320 deadline: 1731025515212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:15,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:15,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:15,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:15,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:15,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741944_1120 (size=14594) 2024-11-08T00:24:15,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025515213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:15,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:15,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57324 deadline: 1731025515214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:15,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:15,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025515216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:15,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:15,226 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:15,226 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:15,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:15,226 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:15,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:15,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:15,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:15,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:15,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:15,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57290 deadline: 1731025515220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:15,227 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:15,227 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:15,227 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:15,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:15,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57320 deadline: 1731025515331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:15,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:15,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025515331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:15,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:15,333 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:15,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:15,333 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:15,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:15,333 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:15,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:15,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025515338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:15,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:15,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:15,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:15,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:15,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57324 deadline: 1731025515347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:15,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:15,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57290 deadline: 1731025515347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:15,349 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:15,349 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:15,349 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:15,352 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:15,352 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:15,352 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:15,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:15,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:15,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025515538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:15,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57320 deadline: 1731025515538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:15,552 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:15,552 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:15,552 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:15,552 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:15,552 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:15,552 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:15,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:15,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025515557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:15,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:15,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57290 deadline: 1731025515558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:15,560 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:15,560 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:15,560 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:15,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:15,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57324 deadline: 1731025515558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:15,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:15,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:15,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:15,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:15,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:15,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:15,625 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:15,633 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108c02b7b5ec1004224bcdc2bbc1501ad71_d9a7fdf8b82889b95543ffed36720442 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108c02b7b5ec1004224bcdc2bbc1501ad71_d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:15,638 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/e00bd41cdad242deb1b41ac16e2e7938, store: [table=TestAcidGuarantees family=A region=d9a7fdf8b82889b95543ffed36720442] 2024-11-08T00:24:15,639 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/e00bd41cdad242deb1b41ac16e2e7938 is 175, key is test_row_0/A:col10/1731025454978/Put/seqid=0 2024-11-08T00:24:15,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741945_1121 (size=39549) 2024-11-08T00:24:15,663 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=56, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/e00bd41cdad242deb1b41ac16e2e7938 2024-11-08T00:24:15,678 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T00:24:15,696 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/ef6065e5dbce4632bf05235592f917c7 is 50, key is test_row_0/B:col10/1731025454978/Put/seqid=0 2024-11-08T00:24:15,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741946_1122 (size=12001) 2024-11-08T00:24:15,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:15,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57320 deadline: 1731025515858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:15,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:15,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:15,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:15,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025515860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:15,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:15,863 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:15,863 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:15,863 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:15,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:15,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57290 deadline: 1731025515867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:15,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:15,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57324 deadline: 1731025515867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:15,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:15,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:15,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:15,869 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:15,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:15,869 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:15,869 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:15,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025515867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:15,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:15,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:15,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:16,140 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/ef6065e5dbce4632bf05235592f917c7 2024-11-08T00:24:16,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/9860599e3f1c49038216cbca85e7060f is 50, key is test_row_0/C:col10/1731025454978/Put/seqid=0 2024-11-08T00:24:16,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741947_1123 (size=12001) 2024-11-08T00:24:16,179 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/9860599e3f1c49038216cbca85e7060f 2024-11-08T00:24:16,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/e00bd41cdad242deb1b41ac16e2e7938 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/e00bd41cdad242deb1b41ac16e2e7938 2024-11-08T00:24:16,195 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/e00bd41cdad242deb1b41ac16e2e7938, entries=200, sequenceid=56, filesize=38.6 K 2024-11-08T00:24:16,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/ef6065e5dbce4632bf05235592f917c7 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/ef6065e5dbce4632bf05235592f917c7 2024-11-08T00:24:16,209 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/ef6065e5dbce4632bf05235592f917c7, entries=150, sequenceid=56, filesize=11.7 K 2024-11-08T00:24:16,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/9860599e3f1c49038216cbca85e7060f as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/9860599e3f1c49038216cbca85e7060f 2024-11-08T00:24:16,218 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/9860599e3f1c49038216cbca85e7060f, entries=150, sequenceid=56, filesize=11.7 K 2024-11-08T00:24:16,219 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for d9a7fdf8b82889b95543ffed36720442 in 1093ms, sequenceid=56, compaction requested=true 2024-11-08T00:24:16,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d9a7fdf8b82889b95543ffed36720442: 2024-11-08T00:24:16,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9a7fdf8b82889b95543ffed36720442:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T00:24:16,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:16,219 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:24:16,219 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:24:16,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9a7fdf8b82889b95543ffed36720442:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T00:24:16,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:16,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9a7fdf8b82889b95543ffed36720442:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:24:16,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:24:16,221 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127237 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:24:16,221 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:24:16,221 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): d9a7fdf8b82889b95543ffed36720442/A is initiating minor compaction (all files) 2024-11-08T00:24:16,221 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): d9a7fdf8b82889b95543ffed36720442/B is initiating minor compaction (all files) 2024-11-08T00:24:16,221 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9a7fdf8b82889b95543ffed36720442/A in TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:16,221 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9a7fdf8b82889b95543ffed36720442/B in TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:16,221 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/ce2b1e6d4cf84bd3803b3aa60b6b6a7b, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/2e23c1ee94804405839ddcab4fe4b70b, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/e00bd41cdad242deb1b41ac16e2e7938] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp, totalSize=124.3 K 2024-11-08T00:24:16,221 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/5089aa09a49243f98b336765b4ff6045, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/6a46e44a3c624f06ae4619b5306e28b1, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/ef6065e5dbce4632bf05235592f917c7] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp, totalSize=35.2 K 2024-11-08T00:24:16,221 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:16,221 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. files: [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/ce2b1e6d4cf84bd3803b3aa60b6b6a7b, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/2e23c1ee94804405839ddcab4fe4b70b, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/e00bd41cdad242deb1b41ac16e2e7938] 2024-11-08T00:24:16,222 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 5089aa09a49243f98b336765b4ff6045, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=19, earliestPutTs=1731025452617 2024-11-08T00:24:16,222 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce2b1e6d4cf84bd3803b3aa60b6b6a7b, keycount=300, bloomtype=ROW, size=55.4 K, encoding=NONE, compression=NONE, seqNum=19, earliestPutTs=1731025452617 2024-11-08T00:24:16,222 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a46e44a3c624f06ae4619b5306e28b1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731025452663 2024-11-08T00:24:16,222 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e23c1ee94804405839ddcab4fe4b70b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731025452663 2024-11-08T00:24:16,222 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting ef6065e5dbce4632bf05235592f917c7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731025454943 2024-11-08T00:24:16,223 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting e00bd41cdad242deb1b41ac16e2e7938, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731025454943 2024-11-08T00:24:16,249 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9a7fdf8b82889b95543ffed36720442#B#compaction#105 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:16,250 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/d474b77f8af245ebb4a2c1f16b6102d1 is 50, key is test_row_0/B:col10/1731025454978/Put/seqid=0 2024-11-08T00:24:16,268 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d9a7fdf8b82889b95543ffed36720442] 2024-11-08T00:24:16,300 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241108ef91ea219d614473a79a762e10f49f2b_d9a7fdf8b82889b95543ffed36720442 store=[table=TestAcidGuarantees family=A region=d9a7fdf8b82889b95543ffed36720442] 2024-11-08T00:24:16,308 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241108ef91ea219d614473a79a762e10f49f2b_d9a7fdf8b82889b95543ffed36720442, store=[table=TestAcidGuarantees family=A region=d9a7fdf8b82889b95543ffed36720442] 2024-11-08T00:24:16,308 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108ef91ea219d614473a79a762e10f49f2b_d9a7fdf8b82889b95543ffed36720442 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d9a7fdf8b82889b95543ffed36720442] 2024-11-08T00:24:16,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741948_1124 (size=12104) 2024-11-08T00:24:16,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741949_1125 (size=4469) 2024-11-08T00:24:16,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:16,380 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:24:16,380 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d9a7fdf8b82889b95543ffed36720442 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-08T00:24:16,412 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110865035815226d495584bbf90b7a9716b1_d9a7fdf8b82889b95543ffed36720442 is 50, key is test_row_0/A:col10/1731025456372/Put/seqid=0 2024-11-08T00:24:16,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:16,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025516419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:16,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:16,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57324 deadline: 1731025516422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:16,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:16,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:16,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:16,426 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:16,426 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:16,426 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:16,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:16,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57320 deadline: 1731025516423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:16,427 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:16,427 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:16,427 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:16,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:16,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57290 deadline: 1731025516426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:16,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:16,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:16,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:16,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:16,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025516426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:16,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:16,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:16,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:16,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741950_1126 (size=14594) 2024-11-08T00:24:16,476 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:16,489 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110865035815226d495584bbf90b7a9716b1_d9a7fdf8b82889b95543ffed36720442 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110865035815226d495584bbf90b7a9716b1_d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:16,491 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/62abf286e4c44f96bda5137b51942cf9, store: [table=TestAcidGuarantees family=A region=d9a7fdf8b82889b95543ffed36720442] 2024-11-08T00:24:16,492 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/62abf286e4c44f96bda5137b51942cf9 is 175, key is test_row_0/A:col10/1731025456372/Put/seqid=0 2024-11-08T00:24:16,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:16,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57324 deadline: 1731025516528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:16,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:16,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:16,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:16,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:16,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025516538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:16,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:16,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:16,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:16,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025516538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:16,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:16,541 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:16,541 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:16,541 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:16,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:16,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57320 deadline: 1731025516538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:16,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:16,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:16,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:16,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:16,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57290 deadline: 1731025516538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:16,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:16,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:16,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:16,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741951_1127 (size=39549) 2024-11-08T00:24:16,735 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9a7fdf8b82889b95543ffed36720442#A#compaction#106 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:16,736 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/d7019f8aba594c529714d6e2fc692aa6 is 175, key is test_row_0/A:col10/1731025454978/Put/seqid=0 2024-11-08T00:24:16,746 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/d474b77f8af245ebb4a2c1f16b6102d1 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/d474b77f8af245ebb4a2c1f16b6102d1 2024-11-08T00:24:16,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:16,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57324 deadline: 1731025516738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:16,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:16,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57320 deadline: 1731025516748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:16,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:16,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025516748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:16,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:16,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57290 deadline: 1731025516748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:16,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:16,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:16,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:16,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:16,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025516748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:16,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:16,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:16,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:16,752 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:16,752 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:16,752 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:16,752 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:16,752 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:16,752 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:16,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:16,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:16,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:16,757 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9a7fdf8b82889b95543ffed36720442/B of d9a7fdf8b82889b95543ffed36720442 into d474b77f8af245ebb4a2c1f16b6102d1(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:16,758 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9a7fdf8b82889b95543ffed36720442: 2024-11-08T00:24:16,758 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., storeName=d9a7fdf8b82889b95543ffed36720442/B, priority=13, startTime=1731025456219; duration=0sec 2024-11-08T00:24:16,758 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:24:16,758 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9a7fdf8b82889b95543ffed36720442:B 2024-11-08T00:24:16,758 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:24:16,759 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:24:16,759 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): d9a7fdf8b82889b95543ffed36720442/C is initiating minor compaction (all files) 2024-11-08T00:24:16,759 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9a7fdf8b82889b95543ffed36720442/C in TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:16,759 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/644f453fa6b14d8baf64468e5ca7c77e, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/22d123ca509a48b4ae2398eda8759ae3, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/9860599e3f1c49038216cbca85e7060f] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp, totalSize=35.2 K 2024-11-08T00:24:16,760 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 644f453fa6b14d8baf64468e5ca7c77e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=19, earliestPutTs=1731025452617 2024-11-08T00:24:16,765 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 22d123ca509a48b4ae2398eda8759ae3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731025452663 2024-11-08T00:24:16,765 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 9860599e3f1c49038216cbca85e7060f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731025454943 2024-11-08T00:24:16,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741952_1128 (size=31058) 2024-11-08T00:24:16,782 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9a7fdf8b82889b95543ffed36720442#C#compaction#108 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:16,783 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/c20254017fcc4c4f8fe32e5e7732cddb is 50, key is test_row_0/C:col10/1731025454978/Put/seqid=0 2024-11-08T00:24:16,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-08T00:24:16,798 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:24:16,798 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-08T00:24:16,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-08T00:24:16,801 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T00:24:16,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-08T00:24:16,803 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T00:24:16,803 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T00:24:16,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741953_1129 (size=12104) 2024-11-08T00:24:16,816 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/c20254017fcc4c4f8fe32e5e7732cddb as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/c20254017fcc4c4f8fe32e5e7732cddb 2024-11-08T00:24:16,823 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9a7fdf8b82889b95543ffed36720442/C of d9a7fdf8b82889b95543ffed36720442 into c20254017fcc4c4f8fe32e5e7732cddb(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:16,823 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9a7fdf8b82889b95543ffed36720442: 2024-11-08T00:24:16,823 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., storeName=d9a7fdf8b82889b95543ffed36720442/C, priority=13, startTime=1731025456220; duration=0sec 2024-11-08T00:24:16,823 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:16,823 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9a7fdf8b82889b95543ffed36720442:C 2024-11-08T00:24:16,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-08T00:24:16,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-08T00:24:16,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:16,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. as already flushing 2024-11-08T00:24:16,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:16,961 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:16,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:16,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:16,969 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=79, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/62abf286e4c44f96bda5137b51942cf9 2024-11-08T00:24:16,998 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/8d14730a23814202b8c6ca69e2a66cd5 is 50, key is test_row_0/B:col10/1731025456372/Put/seqid=0 2024-11-08T00:24:17,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741954_1130 (size=12001) 2024-11-08T00:24:17,036 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/8d14730a23814202b8c6ca69e2a66cd5 2024-11-08T00:24:17,060 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/e056401ee9404ba285a3ed1f5498a85d is 50, key is test_row_0/C:col10/1731025456372/Put/seqid=0 2024-11-08T00:24:17,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:17,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57324 deadline: 1731025517057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:17,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:17,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:17,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:17,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:17,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57320 deadline: 1731025517057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:17,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:17,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:17,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:17,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:17,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025517058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:17,071 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:17,071 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:17,071 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:17,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:17,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:17,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57290 deadline: 1731025517058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:17,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025517058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:17,072 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:17,072 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:17,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:17,072 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:17,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:17,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:17,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741955_1131 (size=12001) 2024-11-08T00:24:17,109 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/e056401ee9404ba285a3ed1f5498a85d 2024-11-08T00:24:17,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-08T00:24:17,118 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-08T00:24:17,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:17,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. as already flushing 2024-11-08T00:24:17,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:17,120 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:17,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:17,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:17,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/62abf286e4c44f96bda5137b51942cf9 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/62abf286e4c44f96bda5137b51942cf9 2024-11-08T00:24:17,131 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/62abf286e4c44f96bda5137b51942cf9, entries=200, sequenceid=79, filesize=38.6 K 2024-11-08T00:24:17,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/8d14730a23814202b8c6ca69e2a66cd5 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/8d14730a23814202b8c6ca69e2a66cd5 2024-11-08T00:24:17,142 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/8d14730a23814202b8c6ca69e2a66cd5, entries=150, sequenceid=79, filesize=11.7 K 2024-11-08T00:24:17,144 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/e056401ee9404ba285a3ed1f5498a85d as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/e056401ee9404ba285a3ed1f5498a85d 2024-11-08T00:24:17,150 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/e056401ee9404ba285a3ed1f5498a85d, entries=150, sequenceid=79, filesize=11.7 K 2024-11-08T00:24:17,164 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for d9a7fdf8b82889b95543ffed36720442 in 784ms, sequenceid=79, compaction requested=false 2024-11-08T00:24:17,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d9a7fdf8b82889b95543ffed36720442: 2024-11-08T00:24:17,175 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/d7019f8aba594c529714d6e2fc692aa6 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/d7019f8aba594c529714d6e2fc692aa6 2024-11-08T00:24:17,197 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9a7fdf8b82889b95543ffed36720442/A of d9a7fdf8b82889b95543ffed36720442 into d7019f8aba594c529714d6e2fc692aa6(size=30.3 K), total size for store is 69.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:17,197 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9a7fdf8b82889b95543ffed36720442: 2024-11-08T00:24:17,197 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., storeName=d9a7fdf8b82889b95543ffed36720442/A, priority=13, startTime=1731025456219; duration=0sec 2024-11-08T00:24:17,197 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:17,197 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9a7fdf8b82889b95543ffed36720442:A 2024-11-08T00:24:17,277 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-08T00:24:17,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:17,278 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing d9a7fdf8b82889b95543ffed36720442 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-08T00:24:17,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108f836d42cecfe43179ae86f9de3ee40db_d9a7fdf8b82889b95543ffed36720442 is 50, key is test_row_0/A:col10/1731025456422/Put/seqid=0 2024-11-08T00:24:17,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741956_1132 (size=12154) 2024-11-08T00:24:17,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:17,411 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108f836d42cecfe43179ae86f9de3ee40db_d9a7fdf8b82889b95543ffed36720442 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108f836d42cecfe43179ae86f9de3ee40db_d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:17,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/d7353164bca944c583fd746be5e6b983, store: [table=TestAcidGuarantees family=A region=d9a7fdf8b82889b95543ffed36720442] 2024-11-08T00:24:17,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/d7353164bca944c583fd746be5e6b983 is 175, key is test_row_0/A:col10/1731025456422/Put/seqid=0 2024-11-08T00:24:17,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-08T00:24:17,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741957_1133 (size=30955) 2024-11-08T00:24:17,473 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/d7353164bca944c583fd746be5e6b983 2024-11-08T00:24:17,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/2bba88877d7648dbac4de8fa5761b813 is 50, key is test_row_0/B:col10/1731025456422/Put/seqid=0 2024-11-08T00:24:17,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741958_1134 (size=12001) 2024-11-08T00:24:17,540 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/2bba88877d7648dbac4de8fa5761b813 2024-11-08T00:24:17,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/eb72ee21e11241ca8e8336d9318697a0 is 50, key is test_row_0/C:col10/1731025456422/Put/seqid=0 2024-11-08T00:24:17,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. as already flushing 2024-11-08T00:24:17,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:17,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741959_1135 (size=12001) 2024-11-08T00:24:17,612 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/eb72ee21e11241ca8e8336d9318697a0 2024-11-08T00:24:17,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/d7353164bca944c583fd746be5e6b983 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/d7353164bca944c583fd746be5e6b983 2024-11-08T00:24:17,639 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/d7353164bca944c583fd746be5e6b983, entries=150, sequenceid=95, filesize=30.2 K 2024-11-08T00:24:17,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/2bba88877d7648dbac4de8fa5761b813 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/2bba88877d7648dbac4de8fa5761b813 2024-11-08T00:24:17,660 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/2bba88877d7648dbac4de8fa5761b813, entries=150, sequenceid=95, filesize=11.7 K 2024-11-08T00:24:17,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/eb72ee21e11241ca8e8336d9318697a0 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/eb72ee21e11241ca8e8336d9318697a0 2024-11-08T00:24:17,682 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/eb72ee21e11241ca8e8336d9318697a0, entries=150, sequenceid=95, filesize=11.7 K 2024-11-08T00:24:17,698 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for d9a7fdf8b82889b95543ffed36720442 in 419ms, sequenceid=95, compaction requested=true 2024-11-08T00:24:17,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for d9a7fdf8b82889b95543ffed36720442: 2024-11-08T00:24:17,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:17,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-08T00:24:17,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:17,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-08T00:24:17,699 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:24:17,699 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d9a7fdf8b82889b95543ffed36720442 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-08T00:24:17,703 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-08T00:24:17,703 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 897 msec 2024-11-08T00:24:17,706 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 906 msec 2024-11-08T00:24:17,728 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110882d91698e5a645b3a1dddfea859c43f3_d9a7fdf8b82889b95543ffed36720442 is 50, key is test_row_0/A:col10/1731025457695/Put/seqid=0 2024-11-08T00:24:17,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:17,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57324 deadline: 1731025517721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:17,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:17,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57290 deadline: 1731025517720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:17,736 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:17,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:17,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:17,737 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:17,737 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:17,737 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:17,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:17,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025517730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:17,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:17,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57320 deadline: 1731025517735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:17,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:17,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:17,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:17,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:17,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:17,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:17,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:17,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025517736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:17,742 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:17,742 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:17,742 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:17,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741960_1136 (size=17034) 2024-11-08T00:24:17,781 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:17,787 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110882d91698e5a645b3a1dddfea859c43f3_d9a7fdf8b82889b95543ffed36720442 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110882d91698e5a645b3a1dddfea859c43f3_d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:17,789 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/f8bf53a234b441429bff84389f948ad7, store: [table=TestAcidGuarantees family=A region=d9a7fdf8b82889b95543ffed36720442] 2024-11-08T00:24:17,790 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/f8bf53a234b441429bff84389f948ad7 is 175, key is test_row_0/A:col10/1731025457695/Put/seqid=0 2024-11-08T00:24:17,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741961_1137 (size=48139) 2024-11-08T00:24:17,809 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=120, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/f8bf53a234b441429bff84389f948ad7 2024-11-08T00:24:17,830 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/2e0cc5a1dad64d3aafa01511ff3fc576 is 50, key is test_row_0/B:col10/1731025457695/Put/seqid=0 2024-11-08T00:24:17,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:17,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57324 deadline: 1731025517847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:17,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:17,853 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:17,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57320 deadline: 1731025517848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:17,853 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:17,853 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:17,854 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:17,854 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:17,854 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:17,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:17,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025517848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:17,854 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:17,855 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:17,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:17,855 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:17,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57290 deadline: 1731025517848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:17,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:17,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:17,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:17,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025517848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:17,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:17,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:17,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:17,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:17,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741962_1138 (size=12001) 2024-11-08T00:24:17,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-08T00:24:17,937 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:24:17,938 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-08T00:24:17,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-08T00:24:17,940 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T00:24:17,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-08T00:24:17,943 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T00:24:17,943 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T00:24:18,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-08T00:24:18,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:18,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:18,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57320 deadline: 1731025518069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:18,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57290 deadline: 1731025518069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:18,075 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:18,075 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:18,075 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:18,075 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:18,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:18,075 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:18,075 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:18,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025518069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:18,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:18,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:18,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:18,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:18,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025518069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:18,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:18,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57324 deadline: 1731025518069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:18,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:18,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:18,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:18,079 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:18,079 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:18,079 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:18,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-08T00:24:18,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:18,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. as already flushing 2024-11-08T00:24:18,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:18,096 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:18,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:18,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:18,250 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-08T00:24:18,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:18,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. as already flushing 2024-11-08T00:24:18,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:18,251 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:18,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:18,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:18,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-08T00:24:18,286 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/2e0cc5a1dad64d3aafa01511ff3fc576 2024-11-08T00:24:18,303 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/5a23e86eb26140179af08c7dc4e5e6b3 is 50, key is test_row_0/C:col10/1731025457695/Put/seqid=0 2024-11-08T00:24:18,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741963_1139 (size=12001) 2024-11-08T00:24:18,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:18,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57290 deadline: 1731025518378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:18,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:18,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:18,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:18,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:18,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025518378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:18,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:18,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:18,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:18,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:18,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57324 deadline: 1731025518387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:18,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:18,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57320 deadline: 1731025518387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:18,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:18,392 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:18,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025518388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:18,392 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:18,392 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:18,392 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:18,393 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:18,393 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:18,394 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:18,394 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:18,394 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:18,404 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-08T00:24:18,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:18,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. as already flushing 2024-11-08T00:24:18,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:18,404 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:18,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:18,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:18,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-08T00:24:18,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-08T00:24:18,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:18,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. as already flushing 2024-11-08T00:24:18,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:18,568 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:18,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:18,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:18,724 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-08T00:24:18,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:18,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. as already flushing 2024-11-08T00:24:18,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:18,725 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:18,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:18,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:18,743 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/5a23e86eb26140179af08c7dc4e5e6b3 2024-11-08T00:24:18,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/f8bf53a234b441429bff84389f948ad7 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/f8bf53a234b441429bff84389f948ad7 2024-11-08T00:24:18,801 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/f8bf53a234b441429bff84389f948ad7, entries=250, sequenceid=120, filesize=47.0 K 2024-11-08T00:24:18,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/2e0cc5a1dad64d3aafa01511ff3fc576 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/2e0cc5a1dad64d3aafa01511ff3fc576 2024-11-08T00:24:18,842 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/2e0cc5a1dad64d3aafa01511ff3fc576, entries=150, sequenceid=120, filesize=11.7 K 2024-11-08T00:24:18,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/5a23e86eb26140179af08c7dc4e5e6b3 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/5a23e86eb26140179af08c7dc4e5e6b3 2024-11-08T00:24:18,881 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/5a23e86eb26140179af08c7dc4e5e6b3, entries=150, sequenceid=120, filesize=11.7 K 2024-11-08T00:24:18,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-08T00:24:18,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:18,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. as already flushing 2024-11-08T00:24:18,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:18,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:18,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:18,889 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for d9a7fdf8b82889b95543ffed36720442 in 1190ms, sequenceid=120, compaction requested=true 2024-11-08T00:24:18,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d9a7fdf8b82889b95543ffed36720442: 2024-11-08T00:24:18,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9a7fdf8b82889b95543ffed36720442:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T00:24:18,889 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T00:24:18,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:18,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9a7fdf8b82889b95543ffed36720442:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T00:24:18,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:24:18,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9a7fdf8b82889b95543ffed36720442:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:24:18,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-08T00:24:18,891 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 149701 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T00:24:18,891 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): d9a7fdf8b82889b95543ffed36720442/A is initiating minor compaction (all files) 2024-11-08T00:24:18,891 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9a7fdf8b82889b95543ffed36720442/A in TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:18,891 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/d7019f8aba594c529714d6e2fc692aa6, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/62abf286e4c44f96bda5137b51942cf9, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/d7353164bca944c583fd746be5e6b983, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/f8bf53a234b441429bff84389f948ad7] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp, totalSize=146.2 K 2024-11-08T00:24:18,891 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:18,891 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. files: [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/d7019f8aba594c529714d6e2fc692aa6, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/62abf286e4c44f96bda5137b51942cf9, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/d7353164bca944c583fd746be5e6b983, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/f8bf53a234b441429bff84389f948ad7] 2024-11-08T00:24:18,892 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T00:24:18,892 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7019f8aba594c529714d6e2fc692aa6, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731025454943 2024-11-08T00:24:18,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:18,893 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62abf286e4c44f96bda5137b51942cf9, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731025455197 2024-11-08T00:24:18,893 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7353164bca944c583fd746be5e6b983, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731025456419 2024-11-08T00:24:18,893 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8bf53a234b441429bff84389f948ad7, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731025457637 2024-11-08T00:24:18,896 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:24:18,896 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d9a7fdf8b82889b95543ffed36720442 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-08T00:24:18,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:18,902 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T00:24:18,902 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): d9a7fdf8b82889b95543ffed36720442/B is initiating minor compaction (all files) 2024-11-08T00:24:18,902 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9a7fdf8b82889b95543ffed36720442/B in TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:18,903 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/d474b77f8af245ebb4a2c1f16b6102d1, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/8d14730a23814202b8c6ca69e2a66cd5, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/2bba88877d7648dbac4de8fa5761b813, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/2e0cc5a1dad64d3aafa01511ff3fc576] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp, totalSize=47.0 K 2024-11-08T00:24:18,904 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting d474b77f8af245ebb4a2c1f16b6102d1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731025454943 2024-11-08T00:24:18,905 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d14730a23814202b8c6ca69e2a66cd5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731025455211 2024-11-08T00:24:18,906 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bba88877d7648dbac4de8fa5761b813, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731025456419 2024-11-08T00:24:18,906 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e0cc5a1dad64d3aafa01511ff3fc576, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731025457637 2024-11-08T00:24:18,912 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d9a7fdf8b82889b95543ffed36720442] 2024-11-08T00:24:18,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108b776d1e602954f2a8a793e27dbc0f37b_d9a7fdf8b82889b95543ffed36720442 is 50, key is test_row_0/A:col10/1731025458894/Put/seqid=0 2024-11-08T00:24:18,941 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110885ce1d9686e5484c814898caa9d35b89_d9a7fdf8b82889b95543ffed36720442 store=[table=TestAcidGuarantees family=A region=d9a7fdf8b82889b95543ffed36720442] 2024-11-08T00:24:18,945 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110885ce1d9686e5484c814898caa9d35b89_d9a7fdf8b82889b95543ffed36720442, store=[table=TestAcidGuarantees family=A region=d9a7fdf8b82889b95543ffed36720442] 2024-11-08T00:24:18,946 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110885ce1d9686e5484c814898caa9d35b89_d9a7fdf8b82889b95543ffed36720442 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d9a7fdf8b82889b95543ffed36720442] 2024-11-08T00:24:18,950 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9a7fdf8b82889b95543ffed36720442#B#compaction#119 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:18,951 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/d943b4a8659e488c83c98344fb5d9901 is 50, key is test_row_0/B:col10/1731025457695/Put/seqid=0 2024-11-08T00:24:19,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741965_1141 (size=4469) 2024-11-08T00:24:19,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741964_1140 (size=14744) 2024-11-08T00:24:19,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:19,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025518994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:19,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:19,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025518998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:19,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:19,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57290 deadline: 1731025519005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:19,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:19,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:19,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:19,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:19,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:19,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:19,017 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:19,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:19,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:19,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:19,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57320 deadline: 1731025519016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:19,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:19,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57324 deadline: 1731025519019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:19,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:19,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:19,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:19,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:19,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:19,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:19,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741966_1142 (size=12241) 2024-11-08T00:24:19,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-08T00:24:19,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:19,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. as already flushing 2024-11-08T00:24:19,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:19,057 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:19,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:19,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:19,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-08T00:24:19,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:19,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57290 deadline: 1731025519128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:19,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:19,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025519130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:19,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:19,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:19,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:19,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:19,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:19,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:19,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:19,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025519130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:19,134 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:19,134 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:19,134 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:19,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:19,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57324 deadline: 1731025519137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:19,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:19,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:19,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:19,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:19,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57320 deadline: 1731025519137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:19,148 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:19,148 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:19,148 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:19,223 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-08T00:24:19,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:19,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. as already flushing 2024-11-08T00:24:19,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:19,223 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:19,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:19,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:19,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:19,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57290 deadline: 1731025519337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:19,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:19,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025519338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:19,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:19,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:19,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025519338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:19,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:19,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:19,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:19,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:19,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:19,341 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:19,341 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:19,341 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:19,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:19,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57324 deadline: 1731025519357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:19,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:19,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57320 deadline: 1731025519357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:19,358 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:19,358 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:19,358 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:19,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:19,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:19,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:19,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-08T00:24:19,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:19,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. as already flushing 2024-11-08T00:24:19,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:19,376 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:19,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:19,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:19,405 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9a7fdf8b82889b95543ffed36720442#A#compaction#117 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:19,406 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/35d1d98eaed844b684170d1f798f4a75 is 175, key is test_row_0/A:col10/1731025457695/Put/seqid=0 2024-11-08T00:24:19,411 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:19,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741967_1143 (size=31195) 2024-11-08T00:24:19,429 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108b776d1e602954f2a8a793e27dbc0f37b_d9a7fdf8b82889b95543ffed36720442 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108b776d1e602954f2a8a793e27dbc0f37b_d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:19,431 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/6520fcbbefda40ab8e67626c26b95942, store: [table=TestAcidGuarantees family=A region=d9a7fdf8b82889b95543ffed36720442] 2024-11-08T00:24:19,433 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/6520fcbbefda40ab8e67626c26b95942 is 175, key is test_row_0/A:col10/1731025458894/Put/seqid=0 2024-11-08T00:24:19,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741968_1144 (size=39699) 2024-11-08T00:24:19,453 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/d943b4a8659e488c83c98344fb5d9901 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/d943b4a8659e488c83c98344fb5d9901 2024-11-08T00:24:19,453 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/6520fcbbefda40ab8e67626c26b95942 2024-11-08T00:24:19,454 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/35d1d98eaed844b684170d1f798f4a75 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/35d1d98eaed844b684170d1f798f4a75 2024-11-08T00:24:19,481 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d9a7fdf8b82889b95543ffed36720442/A of d9a7fdf8b82889b95543ffed36720442 into 35d1d98eaed844b684170d1f798f4a75(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:19,481 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9a7fdf8b82889b95543ffed36720442: 2024-11-08T00:24:19,481 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., storeName=d9a7fdf8b82889b95543ffed36720442/A, priority=12, startTime=1731025458889; duration=0sec 2024-11-08T00:24:19,481 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:24:19,481 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9a7fdf8b82889b95543ffed36720442:A 2024-11-08T00:24:19,481 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T00:24:19,483 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T00:24:19,483 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): d9a7fdf8b82889b95543ffed36720442/C is initiating minor compaction (all files) 2024-11-08T00:24:19,483 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9a7fdf8b82889b95543ffed36720442/C in TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:19,483 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/c20254017fcc4c4f8fe32e5e7732cddb, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/e056401ee9404ba285a3ed1f5498a85d, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/eb72ee21e11241ca8e8336d9318697a0, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/5a23e86eb26140179af08c7dc4e5e6b3] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp, totalSize=47.0 K 2024-11-08T00:24:19,484 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting c20254017fcc4c4f8fe32e5e7732cddb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731025454943 2024-11-08T00:24:19,485 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d9a7fdf8b82889b95543ffed36720442/B of d9a7fdf8b82889b95543ffed36720442 into d943b4a8659e488c83c98344fb5d9901(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:19,485 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9a7fdf8b82889b95543ffed36720442: 2024-11-08T00:24:19,485 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., storeName=d9a7fdf8b82889b95543ffed36720442/B, priority=12, startTime=1731025458889; duration=0sec 2024-11-08T00:24:19,485 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:19,485 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9a7fdf8b82889b95543ffed36720442:B 2024-11-08T00:24:19,485 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting e056401ee9404ba285a3ed1f5498a85d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731025455211 2024-11-08T00:24:19,486 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb72ee21e11241ca8e8336d9318697a0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731025456419 2024-11-08T00:24:19,486 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a23e86eb26140179af08c7dc4e5e6b3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731025457637 2024-11-08T00:24:19,494 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/b498f3901f88434aa5d5e3e0827a75ab is 50, key is test_row_0/B:col10/1731025458894/Put/seqid=0 2024-11-08T00:24:19,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741969_1145 (size=12101) 2024-11-08T00:24:19,514 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/b498f3901f88434aa5d5e3e0827a75ab 2024-11-08T00:24:19,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-08T00:24:19,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:19,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. as already flushing 2024-11-08T00:24:19,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:19,530 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:19,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:19,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:19,535 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9a7fdf8b82889b95543ffed36720442#C#compaction#121 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:19,535 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/3f758ebb38a64836b8be66f3242334db is 50, key is test_row_0/C:col10/1731025457695/Put/seqid=0 2024-11-08T00:24:19,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/b9bd0fc1273547ba995445be0f248cb1 is 50, key is test_row_0/C:col10/1731025458894/Put/seqid=0 2024-11-08T00:24:19,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741970_1146 (size=12241) 2024-11-08T00:24:19,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741971_1147 (size=12101) 2024-11-08T00:24:19,591 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/b9bd0fc1273547ba995445be0f248cb1 2024-11-08T00:24:19,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/6520fcbbefda40ab8e67626c26b95942 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/6520fcbbefda40ab8e67626c26b95942 2024-11-08T00:24:19,617 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/6520fcbbefda40ab8e67626c26b95942, entries=200, sequenceid=132, filesize=38.8 K 2024-11-08T00:24:19,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/b498f3901f88434aa5d5e3e0827a75ab as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/b498f3901f88434aa5d5e3e0827a75ab 2024-11-08T00:24:19,624 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/b498f3901f88434aa5d5e3e0827a75ab, entries=150, sequenceid=132, filesize=11.8 K 2024-11-08T00:24:19,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/b9bd0fc1273547ba995445be0f248cb1 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/b9bd0fc1273547ba995445be0f248cb1 2024-11-08T00:24:19,632 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/b9bd0fc1273547ba995445be0f248cb1, entries=150, sequenceid=132, filesize=11.8 K 2024-11-08T00:24:19,633 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for d9a7fdf8b82889b95543ffed36720442 in 737ms, sequenceid=132, compaction requested=false 2024-11-08T00:24:19,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d9a7fdf8b82889b95543ffed36720442: 2024-11-08T00:24:19,652 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:24:19,652 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d9a7fdf8b82889b95543ffed36720442 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-08T00:24:19,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:19,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108bf8f22b38b7f4130b0cdefa72899cb28_d9a7fdf8b82889b95543ffed36720442 is 50, key is test_row_0/A:col10/1731025458990/Put/seqid=0 2024-11-08T00:24:19,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:19,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025519664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:19,677 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:19,677 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:19,677 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:19,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:19,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57324 deadline: 1731025519667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:19,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:19,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:19,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:19,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:19,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57320 deadline: 1731025519667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:19,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:19,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:19,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:19,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:19,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57290 deadline: 1731025519673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:19,680 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:19,680 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:19,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:19,680 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:19,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025519675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:19,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:19,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:19,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:19,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-08T00:24:19,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:19,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. as already flushing 2024-11-08T00:24:19,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:19,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:19,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:19,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:19,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741972_1148 (size=12304) 2024-11-08T00:24:19,726 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:19,733 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108bf8f22b38b7f4130b0cdefa72899cb28_d9a7fdf8b82889b95543ffed36720442 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108bf8f22b38b7f4130b0cdefa72899cb28_d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:19,734 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/f2770cfa6d93487081c8d4307d34c30a, store: [table=TestAcidGuarantees family=A region=d9a7fdf8b82889b95543ffed36720442] 2024-11-08T00:24:19,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/f2770cfa6d93487081c8d4307d34c30a is 175, key is test_row_0/A:col10/1731025458990/Put/seqid=0 2024-11-08T00:24:19,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:19,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025519787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:19,792 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:19,792 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:19,792 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:19,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:19,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57290 deadline: 1731025519788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:19,793 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:19,793 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:19,793 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:19,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:19,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025519788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:19,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741973_1149 (size=31105) 2024-11-08T00:24:19,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:19,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:19,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:19,838 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-08T00:24:19,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:19,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. as already flushing 2024-11-08T00:24:19,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:19,839 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:19,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:19,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:19,987 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/3f758ebb38a64836b8be66f3242334db as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/3f758ebb38a64836b8be66f3242334db 2024-11-08T00:24:19,995 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d9a7fdf8b82889b95543ffed36720442/C of d9a7fdf8b82889b95543ffed36720442 into 3f758ebb38a64836b8be66f3242334db(size=12.0 K), total size for store is 23.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:19,995 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9a7fdf8b82889b95543ffed36720442: 2024-11-08T00:24:19,995 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., storeName=d9a7fdf8b82889b95543ffed36720442/C, priority=12, startTime=1731025458889; duration=0sec 2024-11-08T00:24:19,996 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:19,996 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9a7fdf8b82889b95543ffed36720442:C 2024-11-08T00:24:19,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-08T00:24:19,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:19,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. as already flushing 2024-11-08T00:24:19,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:19,997 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:19,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:19,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:20,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:20,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57290 deadline: 1731025519999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:20,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:20,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025519999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:20,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:20,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:20,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:20,003 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:20,003 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:20,003 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:20,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:20,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025520007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:20,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:20,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:20,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:20,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-08T00:24:20,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-08T00:24:20,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:20,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. as already flushing 2024-11-08T00:24:20,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:20,151 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:20,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:20,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:20,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:20,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57324 deadline: 1731025520193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:20,195 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:20,195 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:20,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:20,196 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=160, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/f2770cfa6d93487081c8d4307d34c30a 2024-11-08T00:24:20,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:20,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57320 deadline: 1731025520194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:20,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:20,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:20,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:20,218 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/7496209e0fb644b89f7a97478718c5cb is 50, key is test_row_0/B:col10/1731025458990/Put/seqid=0 2024-11-08T00:24:20,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741974_1150 (size=12151) 2024-11-08T00:24:20,305 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-08T00:24:20,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:20,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. as already flushing 2024-11-08T00:24:20,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:20,306 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:20,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:20,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:20,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:20,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025520308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:20,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:20,318 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:20,318 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:20,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57290 deadline: 1731025520308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:20,318 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:20,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:20,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:20,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:20,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:20,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025520317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:20,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:20,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:20,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:20,459 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-08T00:24:20,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:20,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. as already flushing 2024-11-08T00:24:20,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:20,459 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:20,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:20,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:20,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-08T00:24:20,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:20,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. as already flushing 2024-11-08T00:24:20,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:20,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:20,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:20,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:20,640 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/7496209e0fb644b89f7a97478718c5cb 2024-11-08T00:24:20,650 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/18d4731da1724941a326c96136344118 is 50, key is test_row_0/C:col10/1731025458990/Put/seqid=0 2024-11-08T00:24:20,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741975_1151 (size=12151) 2024-11-08T00:24:20,671 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/18d4731da1724941a326c96136344118 2024-11-08T00:24:20,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/f2770cfa6d93487081c8d4307d34c30a as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/f2770cfa6d93487081c8d4307d34c30a 2024-11-08T00:24:20,685 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/f2770cfa6d93487081c8d4307d34c30a, entries=150, sequenceid=160, filesize=30.4 K 2024-11-08T00:24:20,688 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/7496209e0fb644b89f7a97478718c5cb as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/7496209e0fb644b89f7a97478718c5cb 2024-11-08T00:24:20,695 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/7496209e0fb644b89f7a97478718c5cb, entries=150, sequenceid=160, filesize=11.9 K 2024-11-08T00:24:20,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/18d4731da1724941a326c96136344118 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/18d4731da1724941a326c96136344118 2024-11-08T00:24:20,706 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/18d4731da1724941a326c96136344118, entries=150, sequenceid=160, filesize=11.9 K 2024-11-08T00:24:20,707 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for d9a7fdf8b82889b95543ffed36720442 in 1054ms, sequenceid=160, compaction requested=true 2024-11-08T00:24:20,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d9a7fdf8b82889b95543ffed36720442: 2024-11-08T00:24:20,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9a7fdf8b82889b95543ffed36720442:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T00:24:20,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:20,707 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:24:20,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9a7fdf8b82889b95543ffed36720442:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T00:24:20,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:20,707 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:24:20,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9a7fdf8b82889b95543ffed36720442:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:24:20,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:24:20,710 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:24:20,710 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): d9a7fdf8b82889b95543ffed36720442/A is initiating minor compaction (all files) 2024-11-08T00:24:20,710 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9a7fdf8b82889b95543ffed36720442/A in TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:20,710 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/35d1d98eaed844b684170d1f798f4a75, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/6520fcbbefda40ab8e67626c26b95942, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/f2770cfa6d93487081c8d4307d34c30a] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp, totalSize=99.6 K 2024-11-08T00:24:20,710 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:20,710 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. files: [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/35d1d98eaed844b684170d1f798f4a75, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/6520fcbbefda40ab8e67626c26b95942, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/f2770cfa6d93487081c8d4307d34c30a] 2024-11-08T00:24:20,711 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36493 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:24:20,711 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35d1d98eaed844b684170d1f798f4a75, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731025457637 2024-11-08T00:24:20,711 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): d9a7fdf8b82889b95543ffed36720442/B is initiating minor compaction (all files) 2024-11-08T00:24:20,711 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9a7fdf8b82889b95543ffed36720442/B in TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:20,711 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/d943b4a8659e488c83c98344fb5d9901, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/b498f3901f88434aa5d5e3e0827a75ab, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/7496209e0fb644b89f7a97478718c5cb] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp, totalSize=35.6 K 2024-11-08T00:24:20,712 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6520fcbbefda40ab8e67626c26b95942, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731025457713 2024-11-08T00:24:20,712 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting d943b4a8659e488c83c98344fb5d9901, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731025457637 2024-11-08T00:24:20,712 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2770cfa6d93487081c8d4307d34c30a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731025458990 2024-11-08T00:24:20,712 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting b498f3901f88434aa5d5e3e0827a75ab, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731025457713 2024-11-08T00:24:20,713 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 7496209e0fb644b89f7a97478718c5cb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731025458990 2024-11-08T00:24:20,723 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9a7fdf8b82889b95543ffed36720442#B#compaction#126 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:20,724 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/0ef2538400844a77830924382dead922 is 50, key is test_row_0/B:col10/1731025458990/Put/seqid=0 2024-11-08T00:24:20,730 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d9a7fdf8b82889b95543ffed36720442] 2024-11-08T00:24:20,745 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241108a3c2f422d1ec416e8a476fb092332199_d9a7fdf8b82889b95543ffed36720442 store=[table=TestAcidGuarantees family=A region=d9a7fdf8b82889b95543ffed36720442] 2024-11-08T00:24:20,748 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241108a3c2f422d1ec416e8a476fb092332199_d9a7fdf8b82889b95543ffed36720442, store=[table=TestAcidGuarantees family=A region=d9a7fdf8b82889b95543ffed36720442] 2024-11-08T00:24:20,748 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108a3c2f422d1ec416e8a476fb092332199_d9a7fdf8b82889b95543ffed36720442 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d9a7fdf8b82889b95543ffed36720442] 2024-11-08T00:24:20,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741976_1152 (size=12493) 2024-11-08T00:24:20,768 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/0ef2538400844a77830924382dead922 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/0ef2538400844a77830924382dead922 2024-11-08T00:24:20,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-08T00:24:20,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:20,773 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing d9a7fdf8b82889b95543ffed36720442 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-08T00:24:20,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108eec4bfd559c74690bc57a84fa3e5e675_d9a7fdf8b82889b95543ffed36720442 is 50, key is test_row_0/A:col10/1731025459673/Put/seqid=0 2024-11-08T00:24:20,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741977_1153 (size=4469) 2024-11-08T00:24:20,791 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9a7fdf8b82889b95543ffed36720442/B of d9a7fdf8b82889b95543ffed36720442 into 0ef2538400844a77830924382dead922(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:20,791 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9a7fdf8b82889b95543ffed36720442: 2024-11-08T00:24:20,791 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., storeName=d9a7fdf8b82889b95543ffed36720442/B, priority=13, startTime=1731025460707; duration=0sec 2024-11-08T00:24:20,791 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:24:20,791 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9a7fdf8b82889b95543ffed36720442:B 2024-11-08T00:24:20,791 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:24:20,792 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36493 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:24:20,793 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): d9a7fdf8b82889b95543ffed36720442/C is initiating minor compaction (all files) 2024-11-08T00:24:20,793 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9a7fdf8b82889b95543ffed36720442/C in TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:20,793 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/3f758ebb38a64836b8be66f3242334db, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/b9bd0fc1273547ba995445be0f248cb1, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/18d4731da1724941a326c96136344118] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp, totalSize=35.6 K 2024-11-08T00:24:20,800 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f758ebb38a64836b8be66f3242334db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731025457637 2024-11-08T00:24:20,804 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting b9bd0fc1273547ba995445be0f248cb1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731025457713 2024-11-08T00:24:20,805 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 18d4731da1724941a326c96136344118, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731025458990 2024-11-08T00:24:20,812 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9a7fdf8b82889b95543ffed36720442#A#compaction#127 average throughput is 0.30 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:20,813 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/c20f3863e3f54ed6b985d2d173c2071a is 175, key is test_row_0/A:col10/1731025458990/Put/seqid=0 2024-11-08T00:24:20,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. as already flushing 2024-11-08T00:24:20,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:20,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741978_1154 (size=12304) 2024-11-08T00:24:20,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:20,847 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108eec4bfd559c74690bc57a84fa3e5e675_d9a7fdf8b82889b95543ffed36720442 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108eec4bfd559c74690bc57a84fa3e5e675_d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:20,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/c3c92059e64b4824acf49903c9afa910, store: [table=TestAcidGuarantees family=A region=d9a7fdf8b82889b95543ffed36720442] 2024-11-08T00:24:20,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/c3c92059e64b4824acf49903c9afa910 is 175, key is test_row_0/A:col10/1731025459673/Put/seqid=0 2024-11-08T00:24:20,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741979_1155 (size=31447) 2024-11-08T00:24:20,863 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9a7fdf8b82889b95543ffed36720442#C#compaction#129 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:20,864 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/0e12d735e53a4afb9b6f6fbbc7c037fe is 50, key is test_row_0/C:col10/1731025458990/Put/seqid=0 2024-11-08T00:24:20,874 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/c20f3863e3f54ed6b985d2d173c2071a as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/c20f3863e3f54ed6b985d2d173c2071a 2024-11-08T00:24:20,910 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9a7fdf8b82889b95543ffed36720442/A of d9a7fdf8b82889b95543ffed36720442 into c20f3863e3f54ed6b985d2d173c2071a(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:20,911 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9a7fdf8b82889b95543ffed36720442: 2024-11-08T00:24:20,911 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., storeName=d9a7fdf8b82889b95543ffed36720442/A, priority=13, startTime=1731025460707; duration=0sec 2024-11-08T00:24:20,911 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:20,911 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9a7fdf8b82889b95543ffed36720442:A 2024-11-08T00:24:20,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741980_1156 (size=31105) 2024-11-08T00:24:20,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741981_1157 (size=12493) 2024-11-08T00:24:20,920 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=171, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/c3c92059e64b4824acf49903c9afa910 2024-11-08T00:24:20,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:20,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025520933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:20,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:20,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025520934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:20,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:20,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:20,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:20,944 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:20,944 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:20,944 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:20,949 INFO [master/3302f0f507bd:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-08T00:24:20,949 INFO [master/3302f0f507bd:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-08T00:24:20,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:20,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57290 deadline: 1731025520937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:20,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:20,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:20,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:20,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/562fe23eef324d5291caf07717ee2536 is 50, key is test_row_0/B:col10/1731025459673/Put/seqid=0 2024-11-08T00:24:21,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741982_1158 (size=12151) 2024-11-08T00:24:21,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:21,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025521048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:21,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:21,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025521049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:21,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:21,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:21,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:21,056 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:21,056 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:21,056 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:21,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:21,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57290 deadline: 1731025521064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:21,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:21,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:21,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:21,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:21,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57324 deadline: 1731025521208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:21,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:21,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57320 deadline: 1731025521211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:21,215 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 3302f0f507bd,40479,1731025415856 for 'test_row_0' in d9a7fdf8b82889b95543ffed36720442 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T00:24:21,216 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:21,216 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:21,216 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:21,216 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 3302f0f507bd,40479,1731025415856 for 'test_row_1' in d9a7fdf8b82889b95543ffed36720442 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T00:24:21,245 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:21,245 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:21,245 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:21,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:21,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025521270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:21,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:21,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:21,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:21,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:21,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025521270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:21,275 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:21,275 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:21,275 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:21,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:21,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57290 deadline: 1731025521277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:21,280 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:21,280 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:21,280 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:21,332 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/0e12d735e53a4afb9b6f6fbbc7c037fe as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/0e12d735e53a4afb9b6f6fbbc7c037fe 2024-11-08T00:24:21,354 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9a7fdf8b82889b95543ffed36720442/C of d9a7fdf8b82889b95543ffed36720442 into 0e12d735e53a4afb9b6f6fbbc7c037fe(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:21,354 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9a7fdf8b82889b95543ffed36720442: 2024-11-08T00:24:21,354 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., storeName=d9a7fdf8b82889b95543ffed36720442/C, priority=13, startTime=1731025460707; duration=0sec 2024-11-08T00:24:21,355 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:21,355 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9a7fdf8b82889b95543ffed36720442:C 2024-11-08T00:24:21,445 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/562fe23eef324d5291caf07717ee2536 2024-11-08T00:24:21,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/8e89857f066b49949897234ef98b4867 is 50, key is test_row_0/C:col10/1731025459673/Put/seqid=0 2024-11-08T00:24:21,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741983_1159 (size=12151) 2024-11-08T00:24:21,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:21,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025521588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:21,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:21,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025521588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:21,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:21,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:21,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:21,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:21,591 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:21,591 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:21,591 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:21,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57290 deadline: 1731025521588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:21,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:21,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:21,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:21,900 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/8e89857f066b49949897234ef98b4867 2024-11-08T00:24:21,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/c3c92059e64b4824acf49903c9afa910 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/c3c92059e64b4824acf49903c9afa910 2024-11-08T00:24:21,957 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/c3c92059e64b4824acf49903c9afa910, entries=150, sequenceid=171, filesize=30.4 K 2024-11-08T00:24:21,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/562fe23eef324d5291caf07717ee2536 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/562fe23eef324d5291caf07717ee2536 2024-11-08T00:24:21,971 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/562fe23eef324d5291caf07717ee2536, entries=150, sequenceid=171, filesize=11.9 K 2024-11-08T00:24:21,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/8e89857f066b49949897234ef98b4867 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/8e89857f066b49949897234ef98b4867 2024-11-08T00:24:21,980 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/8e89857f066b49949897234ef98b4867, entries=150, sequenceid=171, filesize=11.9 K 2024-11-08T00:24:21,993 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for d9a7fdf8b82889b95543ffed36720442 in 1220ms, sequenceid=171, compaction requested=false 2024-11-08T00:24:21,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for d9a7fdf8b82889b95543ffed36720442: 2024-11-08T00:24:21,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:21,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-08T00:24:21,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-08T00:24:21,998 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-08T00:24:21,998 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.0520 sec 2024-11-08T00:24:22,000 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 4.0610 sec 2024-11-08T00:24:22,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-08T00:24:22,108 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:24:22,108 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-08T00:24:22,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-08T00:24:22,110 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:24:22,110 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d9a7fdf8b82889b95543ffed36720442 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-08T00:24:22,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:22,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-08T00:24:22,111 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T00:24:22,112 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T00:24:22,112 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T00:24:22,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:22,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57290 deadline: 1731025522123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:22,128 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:22,128 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:22,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:22,128 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:22,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025522125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:22,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411088beceee55a47499f88f5ab3da4c70735_d9a7fdf8b82889b95543ffed36720442 is 50, key is test_row_0/A:col10/1731025460882/Put/seqid=0 2024-11-08T00:24:22,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:22,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025522127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:22,132 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:22,132 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:22,132 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:22,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:22,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:22,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:22,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741984_1160 (size=17284) 2024-11-08T00:24:22,149 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:22,156 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411088beceee55a47499f88f5ab3da4c70735_d9a7fdf8b82889b95543ffed36720442 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411088beceee55a47499f88f5ab3da4c70735_d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:22,157 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/3f52825d19434d66bce1b29946082a16, store: [table=TestAcidGuarantees family=A region=d9a7fdf8b82889b95543ffed36720442] 2024-11-08T00:24:22,158 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/3f52825d19434d66bce1b29946082a16 is 175, key is test_row_0/A:col10/1731025460882/Put/seqid=0 2024-11-08T00:24:22,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741985_1161 (size=48389) 2024-11-08T00:24:22,177 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=202, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/3f52825d19434d66bce1b29946082a16 2024-11-08T00:24:22,197 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/ad6999aa49f84559b4cf1520f8f057e7 is 50, key is test_row_0/B:col10/1731025460882/Put/seqid=0 2024-11-08T00:24:22,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741986_1162 (size=12151) 2024-11-08T00:24:22,220 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/ad6999aa49f84559b4cf1520f8f057e7 2024-11-08T00:24:22,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-08T00:24:22,236 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/9a860f4c625c471c9f5ae0bef8aef4f4 is 50, key is test_row_0/C:col10/1731025460882/Put/seqid=0 2024-11-08T00:24:22,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:22,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025522237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:22,238 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:22,238 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:22,238 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:22,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:22,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025522237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:22,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:22,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:22,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:22,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:22,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57290 deadline: 1731025522238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:22,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:22,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:22,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:22,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741987_1163 (size=12151) 2024-11-08T00:24:22,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-08T00:24:22,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:22,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. as already flushing 2024-11-08T00:24:22,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:22,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:22,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:22,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:22,419 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-08T00:24:22,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:22,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. as already flushing 2024-11-08T00:24:22,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:22,420 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:22,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:22,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:22,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-08T00:24:22,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:22,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025522447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:22,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:22,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:22,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57290 deadline: 1731025522447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:22,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:22,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:22,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:22,448 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:22,448 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:22,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025522447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:22,448 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:22,449 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:22,449 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:22,449 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:22,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-08T00:24:22,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:22,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. as already flushing 2024-11-08T00:24:22,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:22,573 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:22,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:22,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:22,655 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/9a860f4c625c471c9f5ae0bef8aef4f4 2024-11-08T00:24:22,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/3f52825d19434d66bce1b29946082a16 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/3f52825d19434d66bce1b29946082a16 2024-11-08T00:24:22,666 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/3f52825d19434d66bce1b29946082a16, entries=250, sequenceid=202, filesize=47.3 K 2024-11-08T00:24:22,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/ad6999aa49f84559b4cf1520f8f057e7 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/ad6999aa49f84559b4cf1520f8f057e7 2024-11-08T00:24:22,671 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/ad6999aa49f84559b4cf1520f8f057e7, entries=150, sequenceid=202, filesize=11.9 K 2024-11-08T00:24:22,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/9a860f4c625c471c9f5ae0bef8aef4f4 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/9a860f4c625c471c9f5ae0bef8aef4f4 2024-11-08T00:24:22,687 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/9a860f4c625c471c9f5ae0bef8aef4f4, entries=150, sequenceid=202, filesize=11.9 K 2024-11-08T00:24:22,689 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for d9a7fdf8b82889b95543ffed36720442 in 579ms, sequenceid=202, compaction requested=true 2024-11-08T00:24:22,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d9a7fdf8b82889b95543ffed36720442: 2024-11-08T00:24:22,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9a7fdf8b82889b95543ffed36720442:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T00:24:22,690 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:24:22,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:22,690 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:24:22,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9a7fdf8b82889b95543ffed36720442:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T00:24:22,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:22,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9a7fdf8b82889b95543ffed36720442:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:24:22,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:24:22,691 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:24:22,691 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): d9a7fdf8b82889b95543ffed36720442/B is initiating minor compaction (all files) 2024-11-08T00:24:22,691 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9a7fdf8b82889b95543ffed36720442/B in TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:22,691 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/0ef2538400844a77830924382dead922, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/562fe23eef324d5291caf07717ee2536, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/ad6999aa49f84559b4cf1520f8f057e7] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp, totalSize=35.9 K 2024-11-08T00:24:22,691 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110941 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:24:22,691 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): d9a7fdf8b82889b95543ffed36720442/A is initiating minor compaction (all files) 2024-11-08T00:24:22,692 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ef2538400844a77830924382dead922, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731025458990 2024-11-08T00:24:22,692 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9a7fdf8b82889b95543ffed36720442/A in TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:22,692 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/c20f3863e3f54ed6b985d2d173c2071a, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/c3c92059e64b4824acf49903c9afa910, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/3f52825d19434d66bce1b29946082a16] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp, totalSize=108.3 K 2024-11-08T00:24:22,692 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:22,692 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. files: [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/c20f3863e3f54ed6b985d2d173c2071a, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/c3c92059e64b4824acf49903c9afa910, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/3f52825d19434d66bce1b29946082a16] 2024-11-08T00:24:22,692 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 562fe23eef324d5291caf07717ee2536, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731025459660 2024-11-08T00:24:22,692 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting c20f3863e3f54ed6b985d2d173c2071a, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731025458990 2024-11-08T00:24:22,692 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting ad6999aa49f84559b4cf1520f8f057e7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731025460882 2024-11-08T00:24:22,693 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3c92059e64b4824acf49903c9afa910, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731025459660 2024-11-08T00:24:22,693 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f52825d19434d66bce1b29946082a16, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731025460882 2024-11-08T00:24:22,700 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d9a7fdf8b82889b95543ffed36720442] 2024-11-08T00:24:22,701 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9a7fdf8b82889b95543ffed36720442#B#compaction#135 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:22,701 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/d447a7ecf3b84904a20c866d1c2e8f9f is 50, key is test_row_0/B:col10/1731025460882/Put/seqid=0 2024-11-08T00:24:22,704 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411088906249a43e54615929b2bb304066e64_d9a7fdf8b82889b95543ffed36720442 store=[table=TestAcidGuarantees family=A region=d9a7fdf8b82889b95543ffed36720442] 2024-11-08T00:24:22,707 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411088906249a43e54615929b2bb304066e64_d9a7fdf8b82889b95543ffed36720442, store=[table=TestAcidGuarantees family=A region=d9a7fdf8b82889b95543ffed36720442] 2024-11-08T00:24:22,707 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411088906249a43e54615929b2bb304066e64_d9a7fdf8b82889b95543ffed36720442 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d9a7fdf8b82889b95543ffed36720442] 2024-11-08T00:24:22,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741988_1164 (size=12595) 2024-11-08T00:24:22,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-08T00:24:22,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741989_1165 (size=4469) 2024-11-08T00:24:22,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:22,726 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing d9a7fdf8b82889b95543ffed36720442 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-08T00:24:22,731 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9a7fdf8b82889b95543ffed36720442#A#compaction#136 average throughput is 0.79 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:22,731 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/2e6e6ce6df934eb4a51468985bcee6df is 175, key is test_row_0/A:col10/1731025460882/Put/seqid=0 2024-11-08T00:24:22,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-08T00:24:22,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108e45c5b6f800849daa849a284a5e4e412_d9a7fdf8b82889b95543ffed36720442 is 50, key is test_row_0/A:col10/1731025462114/Put/seqid=0 2024-11-08T00:24:22,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741990_1166 (size=31549) 2024-11-08T00:24:22,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:22,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. as already flushing 2024-11-08T00:24:22,778 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/2e6e6ce6df934eb4a51468985bcee6df as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/2e6e6ce6df934eb4a51468985bcee6df 2024-11-08T00:24:22,785 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9a7fdf8b82889b95543ffed36720442/A of d9a7fdf8b82889b95543ffed36720442 into 2e6e6ce6df934eb4a51468985bcee6df(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:22,785 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9a7fdf8b82889b95543ffed36720442: 2024-11-08T00:24:22,785 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., storeName=d9a7fdf8b82889b95543ffed36720442/A, priority=13, startTime=1731025462690; duration=0sec 2024-11-08T00:24:22,786 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:24:22,786 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9a7fdf8b82889b95543ffed36720442:A 2024-11-08T00:24:22,786 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:24:22,787 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:24:22,787 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): d9a7fdf8b82889b95543ffed36720442/C is initiating minor compaction (all files) 2024-11-08T00:24:22,787 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9a7fdf8b82889b95543ffed36720442/C in TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:22,787 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/0e12d735e53a4afb9b6f6fbbc7c037fe, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/8e89857f066b49949897234ef98b4867, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/9a860f4c625c471c9f5ae0bef8aef4f4] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp, totalSize=35.9 K 2024-11-08T00:24:22,789 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e12d735e53a4afb9b6f6fbbc7c037fe, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731025458990 2024-11-08T00:24:22,789 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e89857f066b49949897234ef98b4867, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731025459660 2024-11-08T00:24:22,790 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a860f4c625c471c9f5ae0bef8aef4f4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731025460882 2024-11-08T00:24:22,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741991_1167 (size=12304) 2024-11-08T00:24:22,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:22,803 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108e45c5b6f800849daa849a284a5e4e412_d9a7fdf8b82889b95543ffed36720442 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108e45c5b6f800849daa849a284a5e4e412_d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:22,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/b72c455684734950a3d852166c965232, store: [table=TestAcidGuarantees family=A region=d9a7fdf8b82889b95543ffed36720442] 2024-11-08T00:24:22,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/b72c455684734950a3d852166c965232 is 175, key is test_row_0/A:col10/1731025462114/Put/seqid=0 2024-11-08T00:24:22,827 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9a7fdf8b82889b95543ffed36720442#C#compaction#138 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:22,828 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/4a1e6ced5135485c842c43abf273984c is 50, key is test_row_0/C:col10/1731025460882/Put/seqid=0 2024-11-08T00:24:22,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741992_1168 (size=31105) 2024-11-08T00:24:22,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741993_1169 (size=12595) 2024-11-08T00:24:22,851 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/4a1e6ced5135485c842c43abf273984c as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/4a1e6ced5135485c842c43abf273984c 2024-11-08T00:24:22,869 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9a7fdf8b82889b95543ffed36720442/C of d9a7fdf8b82889b95543ffed36720442 into 4a1e6ced5135485c842c43abf273984c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:22,870 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9a7fdf8b82889b95543ffed36720442: 2024-11-08T00:24:22,870 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., storeName=d9a7fdf8b82889b95543ffed36720442/C, priority=13, startTime=1731025462690; duration=0sec 2024-11-08T00:24:22,870 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:22,870 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9a7fdf8b82889b95543ffed36720442:C 2024-11-08T00:24:22,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:22,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025522897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:22,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:22,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:22,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:22,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:22,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025522897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:22,906 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:22,906 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:22,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:22,906 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:22,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57290 deadline: 1731025522897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:22,906 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:22,906 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:22,906 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:23,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:23,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025523008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:23,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:23,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025523008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:23,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:23,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:23,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:23,013 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:23,013 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:23,013 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:23,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:23,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57290 deadline: 1731025523017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:23,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:23,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:23,018 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:23,135 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/d447a7ecf3b84904a20c866d1c2e8f9f as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/d447a7ecf3b84904a20c866d1c2e8f9f 2024-11-08T00:24:23,140 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9a7fdf8b82889b95543ffed36720442/B of d9a7fdf8b82889b95543ffed36720442 into d447a7ecf3b84904a20c866d1c2e8f9f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:23,140 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9a7fdf8b82889b95543ffed36720442: 2024-11-08T00:24:23,140 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., storeName=d9a7fdf8b82889b95543ffed36720442/B, priority=13, startTime=1731025462690; duration=0sec 2024-11-08T00:24:23,141 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:23,141 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9a7fdf8b82889b95543ffed36720442:B 2024-11-08T00:24:23,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:23,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025523217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:23,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:23,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:23,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:23,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025523217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:23,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:23,221 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:23,221 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:23,221 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:23,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:23,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57320 deadline: 1731025523227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:23,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:23,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57290 deadline: 1731025523227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:23,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:23,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:23,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:23,229 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 3302f0f507bd,40479,1731025415856 for 'test_row_0' in d9a7fdf8b82889b95543ffed36720442 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T00:24:23,230 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:23,230 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:23,230 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:23,233 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=210, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/b72c455684734950a3d852166c965232 2024-11-08T00:24:23,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/c67eef1bb65b4e8b97569ed871a3d7a6 is 50, key is test_row_0/B:col10/1731025462114/Put/seqid=0 2024-11-08T00:24:23,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741994_1170 (size=12151) 2024-11-08T00:24:23,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-08T00:24:23,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:23,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57324 deadline: 1731025523267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:23,268 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 3302f0f507bd,40479,1731025415856 for 'test_row_1' in d9a7fdf8b82889b95543ffed36720442 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4251 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T00:24:23,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:23,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:23,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:23,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:23,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:23,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025523528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:23,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025523528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:23,532 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:23,532 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:23,532 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:23,532 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:23,532 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:23,532 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:23,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:23,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57290 deadline: 1731025523537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:23,540 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:23,540 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:23,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:23,648 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/c67eef1bb65b4e8b97569ed871a3d7a6 2024-11-08T00:24:23,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/a66a533b63a74516a34c4ad974467975 is 50, key is test_row_0/C:col10/1731025462114/Put/seqid=0 2024-11-08T00:24:23,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741995_1171 (size=12151) 2024-11-08T00:24:23,680 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/a66a533b63a74516a34c4ad974467975 2024-11-08T00:24:23,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/b72c455684734950a3d852166c965232 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/b72c455684734950a3d852166c965232 2024-11-08T00:24:23,695 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/b72c455684734950a3d852166c965232, entries=150, sequenceid=210, filesize=30.4 K 2024-11-08T00:24:23,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/c67eef1bb65b4e8b97569ed871a3d7a6 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/c67eef1bb65b4e8b97569ed871a3d7a6 2024-11-08T00:24:23,701 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/c67eef1bb65b4e8b97569ed871a3d7a6, entries=150, sequenceid=210, filesize=11.9 K 2024-11-08T00:24:23,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/a66a533b63a74516a34c4ad974467975 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/a66a533b63a74516a34c4ad974467975 2024-11-08T00:24:23,708 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/a66a533b63a74516a34c4ad974467975, entries=150, sequenceid=210, filesize=11.9 K 2024-11-08T00:24:23,709 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=174.43 KB/178620 for d9a7fdf8b82889b95543ffed36720442 in 983ms, sequenceid=210, compaction requested=false 2024-11-08T00:24:23,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for d9a7fdf8b82889b95543ffed36720442: 2024-11-08T00:24:23,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:23,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-08T00:24:23,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-08T00:24:23,713 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-08T00:24:23,713 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5980 sec 2024-11-08T00:24:23,715 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.6050 sec 2024-11-08T00:24:24,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:24,038 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:24:24,038 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d9a7fdf8b82889b95543ffed36720442 3/3 column families, dataSize=181.14 KB heapSize=475.36 KB 2024-11-08T00:24:24,047 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411087097ebcb69dc4dcfbdb699e84135cdd9_d9a7fdf8b82889b95543ffed36720442 is 50, key is test_row_0/A:col10/1731025464037/Put/seqid=0 2024-11-08T00:24:24,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:24,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57290 deadline: 1731025524048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:24,050 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:24,050 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:24,050 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:24,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:24,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025524048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:24,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:24,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:24,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:24,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:24,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025524049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:24,055 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:24,055 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:24,055 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:24,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741996_1172 (size=14794) 2024-11-08T00:24:24,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:24,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025524157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:24,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:24,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:24,161 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:24,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:24,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025524157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:24,162 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:24,162 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:24,162 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:24,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-08T00:24:24,257 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:24:24,257 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-08T00:24:24,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-08T00:24:24,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-08T00:24:24,259 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T00:24:24,260 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T00:24:24,260 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T00:24:24,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-08T00:24:24,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:24,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:24,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025524367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:24,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025524367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:24,371 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:24,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:24,371 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:24,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:24,371 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:24,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:24,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-08T00:24:24,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:24,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. as already flushing 2024-11-08T00:24:24,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:24,413 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:24,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:24,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:24,455 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,459 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411087097ebcb69dc4dcfbdb699e84135cdd9_d9a7fdf8b82889b95543ffed36720442 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411087097ebcb69dc4dcfbdb699e84135cdd9_d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:24,460 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/7cd57b92c73540b1b68b62114ae4eb40, store: [table=TestAcidGuarantees family=A region=d9a7fdf8b82889b95543ffed36720442] 2024-11-08T00:24:24,460 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/7cd57b92c73540b1b68b62114ae4eb40 is 175, key is test_row_0/A:col10/1731025464037/Put/seqid=0 2024-11-08T00:24:24,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741997_1173 (size=39749) 2024-11-08T00:24:24,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-08T00:24:24,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:24,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. as already flushing 2024-11-08T00:24:24,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:24,566 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:24,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:24,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:24,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-08T00:24:24,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:24,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:24,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025524677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:24,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025524677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:24,679 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:24,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:24,679 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:24,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:24,679 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:24,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:24,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-08T00:24:24,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:24,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. as already flushing 2024-11-08T00:24:24,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:24,718 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:24,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:24,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:24,866 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=243, memsize=60.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/7cd57b92c73540b1b68b62114ae4eb40 2024-11-08T00:24:24,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-08T00:24:24,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:24,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. as already flushing 2024-11-08T00:24:24,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:24,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:24,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:24,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:24,876 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/df498e57ef82475c9e2eff000adb55ca is 50, key is test_row_0/B:col10/1731025464037/Put/seqid=0 2024-11-08T00:24:24,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-08T00:24:24,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741998_1174 (size=12151) 2024-11-08T00:24:24,895 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/df498e57ef82475c9e2eff000adb55ca 2024-11-08T00:24:24,906 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/27c19b18d7ff480ba0736c5451e95a1d is 50, key is test_row_0/C:col10/1731025464037/Put/seqid=0 2024-11-08T00:24:24,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741999_1175 (size=12151) 2024-11-08T00:24:24,915 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/27c19b18d7ff480ba0736c5451e95a1d 2024-11-08T00:24:24,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/7cd57b92c73540b1b68b62114ae4eb40 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/7cd57b92c73540b1b68b62114ae4eb40 2024-11-08T00:24:24,938 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/7cd57b92c73540b1b68b62114ae4eb40, entries=200, sequenceid=243, filesize=38.8 K 2024-11-08T00:24:24,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/df498e57ef82475c9e2eff000adb55ca as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/df498e57ef82475c9e2eff000adb55ca 2024-11-08T00:24:24,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,945 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/df498e57ef82475c9e2eff000adb55ca, entries=150, sequenceid=243, filesize=11.9 K 2024-11-08T00:24:24,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/27c19b18d7ff480ba0736c5451e95a1d as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/27c19b18d7ff480ba0736c5451e95a1d 2024-11-08T00:24:24,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,974 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/27c19b18d7ff480ba0736c5451e95a1d, entries=150, sequenceid=243, filesize=11.9 K 2024-11-08T00:24:24,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,976 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=20.13 KB/20610 for d9a7fdf8b82889b95543ffed36720442 in 937ms, sequenceid=243, compaction requested=true 2024-11-08T00:24:24,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d9a7fdf8b82889b95543ffed36720442: 2024-11-08T00:24:24,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,976 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:24:24,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9a7fdf8b82889b95543ffed36720442:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T00:24:24,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:24,977 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:24:24,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9a7fdf8b82889b95543ffed36720442:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T00:24:24,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:24,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9a7fdf8b82889b95543ffed36720442:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:24:24,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:24:24,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,982 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102403 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:24:24,982 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): d9a7fdf8b82889b95543ffed36720442/A is initiating minor compaction (all files) 2024-11-08T00:24:24,982 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9a7fdf8b82889b95543ffed36720442/A in TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:24,982 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/2e6e6ce6df934eb4a51468985bcee6df, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/b72c455684734950a3d852166c965232, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/7cd57b92c73540b1b68b62114ae4eb40] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp, totalSize=100.0 K 2024-11-08T00:24:24,982 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:24,982 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. files: [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/2e6e6ce6df934eb4a51468985bcee6df, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/b72c455684734950a3d852166c965232, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/7cd57b92c73540b1b68b62114ae4eb40] 2024-11-08T00:24:24,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,983 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:24:24,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,983 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): d9a7fdf8b82889b95543ffed36720442/B is initiating minor compaction (all files) 2024-11-08T00:24:24,983 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e6e6ce6df934eb4a51468985bcee6df, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731025460882 2024-11-08T00:24:24,983 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9a7fdf8b82889b95543ffed36720442/B in TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:24,983 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/d447a7ecf3b84904a20c866d1c2e8f9f, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/c67eef1bb65b4e8b97569ed871a3d7a6, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/df498e57ef82475c9e2eff000adb55ca] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp, totalSize=36.0 K 2024-11-08T00:24:24,983 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting b72c455684734950a3d852166c965232, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731025462114 2024-11-08T00:24:24,983 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting d447a7ecf3b84904a20c866d1c2e8f9f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731025460882 2024-11-08T00:24:24,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,984 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7cd57b92c73540b1b68b62114ae4eb40, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731025462882 2024-11-08T00:24:24,984 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting c67eef1bb65b4e8b97569ed871a3d7a6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731025462114 2024-11-08T00:24:24,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,984 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting df498e57ef82475c9e2eff000adb55ca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731025462882 2024-11-08T00:24:24,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,997 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d9a7fdf8b82889b95543ffed36720442] 2024-11-08T00:24:24,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:24,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,001 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241108586a46567e654c6cbd8f1e2eae3449b3_d9a7fdf8b82889b95543ffed36720442 store=[table=TestAcidGuarantees family=A region=d9a7fdf8b82889b95543ffed36720442] 2024-11-08T00:24:25,004 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9a7fdf8b82889b95543ffed36720442#B#compaction#145 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:25,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,005 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/5363e0cf72954198b63b7665a2e65038 is 50, key is test_row_0/B:col10/1731025464037/Put/seqid=0 2024-11-08T00:24:25,006 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241108586a46567e654c6cbd8f1e2eae3449b3_d9a7fdf8b82889b95543ffed36720442, store=[table=TestAcidGuarantees family=A region=d9a7fdf8b82889b95543ffed36720442] 2024-11-08T00:24:25,006 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108586a46567e654c6cbd8f1e2eae3449b3_d9a7fdf8b82889b95543ffed36720442 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d9a7fdf8b82889b95543ffed36720442] 2024-11-08T00:24:25,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742000_1176 (size=12697) 2024-11-08T00:24:25,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742001_1177 (size=4469) 2024-11-08T00:24:25,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,020 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9a7fdf8b82889b95543ffed36720442#A#compaction#144 average throughput is 1.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:25,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,021 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/330af41c337e4eafa720c451ed89da21 is 175, key is test_row_0/A:col10/1731025464037/Put/seqid=0 2024-11-08T00:24:25,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-08T00:24:25,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:25,026 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing d9a7fdf8b82889b95543ffed36720442 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-08T00:24:25,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742002_1178 (size=31651) 2024-11-08T00:24:25,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108981f0662c4614233a1964c70cfa74e0d_d9a7fdf8b82889b95543ffed36720442 is 50, key is test_row_1/A:col10/1731025464048/Put/seqid=0 2024-11-08T00:24:25,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742003_1179 (size=7324) 2024-11-08T00:24:25,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,072 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108981f0662c4614233a1964c70cfa74e0d_d9a7fdf8b82889b95543ffed36720442 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108981f0662c4614233a1964c70cfa74e0d_d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:25,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/59f0605a3b36405795cb377ab8163cb9, store: [table=TestAcidGuarantees family=A region=d9a7fdf8b82889b95543ffed36720442] 2024-11-08T00:24:25,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/59f0605a3b36405795cb377ab8163cb9 is 175, key is test_row_1/A:col10/1731025464048/Put/seqid=0 2024-11-08T00:24:25,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742004_1180 (size=13815) 2024-11-08T00:24:25,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,103 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=249, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/59f0605a3b36405795cb377ab8163cb9 2024-11-08T00:24:25,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/d784fe6365b14a519d92d25e92c5ed67 is 50, key is test_row_1/B:col10/1731025464048/Put/seqid=0 2024-11-08T00:24:25,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742005_1181 (size=7365) 2024-11-08T00:24:25,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:25,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:25,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. as already flushing 2024-11-08T00:24:25,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:25,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57290 deadline: 1731025525220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:25,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:25,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025525222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:25,227 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:25,227 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:25,227 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:25,227 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:25,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:25,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025525223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:25,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:25,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:25,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:25,227 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:25,228 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:25,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:25,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57290 deadline: 1731025525337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:25,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:25,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:25,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025525337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:25,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:25,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:25,340 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:25,340 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:25,340 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:25,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:25,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025525340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:25,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:25,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:25,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:25,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-08T00:24:25,419 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/5363e0cf72954198b63b7665a2e65038 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/5363e0cf72954198b63b7665a2e65038 2024-11-08T00:24:25,443 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9a7fdf8b82889b95543ffed36720442/B of d9a7fdf8b82889b95543ffed36720442 into 5363e0cf72954198b63b7665a2e65038(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:25,443 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9a7fdf8b82889b95543ffed36720442: 2024-11-08T00:24:25,443 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., storeName=d9a7fdf8b82889b95543ffed36720442/B, priority=13, startTime=1731025464976; duration=0sec 2024-11-08T00:24:25,443 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:24:25,443 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9a7fdf8b82889b95543ffed36720442:B 2024-11-08T00:24:25,443 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:24:25,456 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:24:25,456 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): d9a7fdf8b82889b95543ffed36720442/C is initiating minor compaction (all files) 2024-11-08T00:24:25,456 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9a7fdf8b82889b95543ffed36720442/C in TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:25,457 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/4a1e6ced5135485c842c43abf273984c, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/a66a533b63a74516a34c4ad974467975, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/27c19b18d7ff480ba0736c5451e95a1d] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp, totalSize=36.0 K 2024-11-08T00:24:25,464 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a1e6ced5135485c842c43abf273984c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731025460882 2024-11-08T00:24:25,470 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting a66a533b63a74516a34c4ad974467975, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731025462114 2024-11-08T00:24:25,476 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 27c19b18d7ff480ba0736c5451e95a1d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731025462882 2024-11-08T00:24:25,516 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/330af41c337e4eafa720c451ed89da21 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/330af41c337e4eafa720c451ed89da21 2024-11-08T00:24:25,521 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/d784fe6365b14a519d92d25e92c5ed67 2024-11-08T00:24:25,554 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9a7fdf8b82889b95543ffed36720442#C#compaction#148 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:25,555 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/c57d5003ef924f06839ddc3d56dc18bf is 50, key is test_row_0/C:col10/1731025464037/Put/seqid=0 2024-11-08T00:24:25,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:25,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57290 deadline: 1731025525549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:25,558 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9a7fdf8b82889b95543ffed36720442/A of d9a7fdf8b82889b95543ffed36720442 into 330af41c337e4eafa720c451ed89da21(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:25,558 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9a7fdf8b82889b95543ffed36720442: 2024-11-08T00:24:25,558 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., storeName=d9a7fdf8b82889b95543ffed36720442/A, priority=13, startTime=1731025464976; duration=0sec 2024-11-08T00:24:25,558 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:25,558 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:25,558 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:25,558 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:25,558 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9a7fdf8b82889b95543ffed36720442:A 2024-11-08T00:24:25,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:25,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025525549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:25,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:25,559 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:25,559 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:25,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025525549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:25,559 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:25,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:25,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:25,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:25,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/e94e1d87625648cca8cb8f2d7c2d59f4 is 50, key is test_row_1/C:col10/1731025464048/Put/seqid=0 2024-11-08T00:24:25,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742006_1182 (size=12697) 2024-11-08T00:24:25,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742007_1183 (size=7365) 2024-11-08T00:24:25,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:25,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025525868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:25,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:25,874 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:25,874 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:25,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57290 deadline: 1731025525868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:25,874 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:25,875 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:25,875 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:25,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:25,875 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:25,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025525868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:25,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:25,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:25,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:25,983 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/c57d5003ef924f06839ddc3d56dc18bf as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/c57d5003ef924f06839ddc3d56dc18bf 2024-11-08T00:24:25,992 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9a7fdf8b82889b95543ffed36720442/C of d9a7fdf8b82889b95543ffed36720442 into c57d5003ef924f06839ddc3d56dc18bf(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:25,992 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/e94e1d87625648cca8cb8f2d7c2d59f4 2024-11-08T00:24:25,993 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9a7fdf8b82889b95543ffed36720442: 2024-11-08T00:24:25,994 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., storeName=d9a7fdf8b82889b95543ffed36720442/C, priority=13, startTime=1731025464977; duration=0sec 2024-11-08T00:24:25,995 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:25,995 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9a7fdf8b82889b95543ffed36720442:C 2024-11-08T00:24:26,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/59f0605a3b36405795cb377ab8163cb9 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/59f0605a3b36405795cb377ab8163cb9 2024-11-08T00:24:26,007 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/59f0605a3b36405795cb377ab8163cb9, entries=50, sequenceid=249, filesize=13.5 K 2024-11-08T00:24:26,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/d784fe6365b14a519d92d25e92c5ed67 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/d784fe6365b14a519d92d25e92c5ed67 2024-11-08T00:24:26,020 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/d784fe6365b14a519d92d25e92c5ed67, entries=50, sequenceid=249, filesize=7.2 K 2024-11-08T00:24:26,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/e94e1d87625648cca8cb8f2d7c2d59f4 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/e94e1d87625648cca8cb8f2d7c2d59f4 2024-11-08T00:24:26,031 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/e94e1d87625648cca8cb8f2d7c2d59f4, entries=50, sequenceid=249, filesize=7.2 K 2024-11-08T00:24:26,034 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=181.14 KB/185490 for d9a7fdf8b82889b95543ffed36720442 in 1008ms, sequenceid=249, compaction requested=false 2024-11-08T00:24:26,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for d9a7fdf8b82889b95543ffed36720442: 2024-11-08T00:24:26,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:26,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-08T00:24:26,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-08T00:24:26,038 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-08T00:24:26,038 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7760 sec 2024-11-08T00:24:26,045 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.7820 sec 2024-11-08T00:24:26,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:26,385 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:24:26,385 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d9a7fdf8b82889b95543ffed36720442 3/3 column families, dataSize=187.85 KB heapSize=492.94 KB 2024-11-08T00:24:26,400 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108e2213f9287694afd9fa188390bfa4041_d9a7fdf8b82889b95543ffed36720442 is 50, key is test_row_0/A:col10/1731025465222/Put/seqid=0 2024-11-08T00:24:26,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:26,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57290 deadline: 1731025526396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:26,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:26,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:26,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:26,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-08T00:24:26,407 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:24:26,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742008_1184 (size=14994) 2024-11-08T00:24:26,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:26,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025526402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:26,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:26,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025526402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:26,409 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:26,410 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:26,410 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:26,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:26,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:26,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:26,412 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-08T00:24:26,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-08T00:24:26,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-08T00:24:26,416 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T00:24:26,418 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T00:24:26,418 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T00:24:26,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:26,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57290 deadline: 1731025526507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:26,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:26,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:26,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:26,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-08T00:24:26,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:26,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025526517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:26,518 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:26,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:26,518 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:26,518 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:26,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025526517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:26,519 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:26,519 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:26,519 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:26,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-08T00:24:26,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:26,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. as already flushing 2024-11-08T00:24:26,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:26,578 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:26,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:26,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:26,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:26,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57290 deadline: 1731025526717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:26,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:26,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:26,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:26,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-08T00:24:26,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-08T00:24:26,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:26,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:26,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. as already flushing 2024-11-08T00:24:26,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025526727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:26,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:26,730 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:26,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:26,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:26,730 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:26,730 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:26,730 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:26,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025526727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:26,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:26,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:26,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:26,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:26,806 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:26,810 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108e2213f9287694afd9fa188390bfa4041_d9a7fdf8b82889b95543ffed36720442 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108e2213f9287694afd9fa188390bfa4041_d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:26,811 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/165d0a3dbf764391af9708f44230c258, store: [table=TestAcidGuarantees family=A region=d9a7fdf8b82889b95543ffed36720442] 2024-11-08T00:24:26,812 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/165d0a3dbf764391af9708f44230c258 is 175, key is test_row_0/A:col10/1731025465222/Put/seqid=0 2024-11-08T00:24:26,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742009_1185 (size=39949) 2024-11-08T00:24:26,822 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=283, memsize=62.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/165d0a3dbf764391af9708f44230c258 2024-11-08T00:24:26,833 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/758e6a0394b0438180d4692f4eb8cf31 is 50, key is test_row_0/B:col10/1731025465222/Put/seqid=0 2024-11-08T00:24:26,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742010_1186 (size=12301) 2024-11-08T00:24:26,882 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-08T00:24:26,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:26,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. as already flushing 2024-11-08T00:24:26,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:26,882 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:26,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:26,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:27,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:27,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57290 deadline: 1731025527028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:27,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:27,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:27,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:27,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-08T00:24:27,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:27,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. as already flushing 2024-11-08T00:24:27,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:27,036 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:27,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:27,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-08T00:24:27,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:27,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025527037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:27,038 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:27,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:27,038 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:27,038 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:27,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025527037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:27,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:27,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:27,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:27,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:27,191 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-08T00:24:27,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:27,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. as already flushing 2024-11-08T00:24:27,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:27,191 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:27,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:27,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:27,244 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/758e6a0394b0438180d4692f4eb8cf31 2024-11-08T00:24:27,257 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/c34732d20a754ed89640f8bba02d0f2a is 50, key is test_row_0/C:col10/1731025465222/Put/seqid=0 2024-11-08T00:24:27,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742011_1187 (size=12301) 2024-11-08T00:24:27,263 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/c34732d20a754ed89640f8bba02d0f2a 2024-11-08T00:24:27,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/165d0a3dbf764391af9708f44230c258 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/165d0a3dbf764391af9708f44230c258 2024-11-08T00:24:27,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:27,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57320 deadline: 1731025527277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:27,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/165d0a3dbf764391af9708f44230c258, entries=200, sequenceid=283, filesize=39.0 K 2024-11-08T00:24:27,278 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 3302f0f507bd,40479,1731025415856 for 'test_row_0' in d9a7fdf8b82889b95543ffed36720442 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8262 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T00:24:27,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:27,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:27,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:27,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/758e6a0394b0438180d4692f4eb8cf31 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/758e6a0394b0438180d4692f4eb8cf31 2024-11-08T00:24:27,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,285 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/758e6a0394b0438180d4692f4eb8cf31, entries=150, sequenceid=283, filesize=12.0 K 2024-11-08T00:24:27,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/c34732d20a754ed89640f8bba02d0f2a as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/c34732d20a754ed89640f8bba02d0f2a 2024-11-08T00:24:27,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,294 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/c34732d20a754ed89640f8bba02d0f2a, entries=150, sequenceid=283, filesize=12.0 K 2024-11-08T00:24:27,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,296 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~187.85 KB/192360, heapSize ~492.89 KB/504720, currentSize=20.13 KB/20610 for d9a7fdf8b82889b95543ffed36720442 in 911ms, sequenceid=283, compaction requested=true 2024-11-08T00:24:27,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d9a7fdf8b82889b95543ffed36720442: 2024-11-08T00:24:27,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9a7fdf8b82889b95543ffed36720442:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T00:24:27,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:27,296 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:24:27,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,296 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:24:27,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9a7fdf8b82889b95543ffed36720442:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T00:24:27,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:27,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9a7fdf8b82889b95543ffed36720442:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:24:27,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:24:27,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,298 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 85415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:24:27,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,298 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): d9a7fdf8b82889b95543ffed36720442/A is initiating minor compaction (all files) 2024-11-08T00:24:27,298 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9a7fdf8b82889b95543ffed36720442/A in TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:27,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,298 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/330af41c337e4eafa720c451ed89da21, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/59f0605a3b36405795cb377ab8163cb9, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/165d0a3dbf764391af9708f44230c258] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp, totalSize=83.4 K 2024-11-08T00:24:27,298 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:27,298 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. files: [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/330af41c337e4eafa720c451ed89da21, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/59f0605a3b36405795cb377ab8163cb9, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/165d0a3dbf764391af9708f44230c258] 2024-11-08T00:24:27,299 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 32363 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:24:27,299 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): d9a7fdf8b82889b95543ffed36720442/B is initiating minor compaction (all files) 2024-11-08T00:24:27,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,299 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9a7fdf8b82889b95543ffed36720442/B in TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:27,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,299 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/5363e0cf72954198b63b7665a2e65038, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/d784fe6365b14a519d92d25e92c5ed67, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/758e6a0394b0438180d4692f4eb8cf31] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp, totalSize=31.6 K 2024-11-08T00:24:27,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,300 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 5363e0cf72954198b63b7665a2e65038, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731025462882 2024-11-08T00:24:27,300 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 330af41c337e4eafa720c451ed89da21, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731025462882 2024-11-08T00:24:27,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,301 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting d784fe6365b14a519d92d25e92c5ed67, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731025464048 2024-11-08T00:24:27,301 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59f0605a3b36405795cb377ab8163cb9, keycount=50, bloomtype=ROW, size=13.5 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731025464048 2024-11-08T00:24:27,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,304 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 758e6a0394b0438180d4692f4eb8cf31, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731025465214 2024-11-08T00:24:27,305 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 165d0a3dbf764391af9708f44230c258, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731025465214 2024-11-08T00:24:27,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,315 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d9a7fdf8b82889b95543ffed36720442] 2024-11-08T00:24:27,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,327 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9a7fdf8b82889b95543ffed36720442#B#compaction#154 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:27,328 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/0cf7fb5e11004dce9e3c6f4db1d2feb4 is 50, key is test_row_0/B:col10/1731025465222/Put/seqid=0 2024-11-08T00:24:27,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,330 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241108b916035a867d4a9eacf8cd84ab62a5e9_d9a7fdf8b82889b95543ffed36720442 store=[table=TestAcidGuarantees family=A region=d9a7fdf8b82889b95543ffed36720442] 2024-11-08T00:24:27,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,332 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241108b916035a867d4a9eacf8cd84ab62a5e9_d9a7fdf8b82889b95543ffed36720442, store=[table=TestAcidGuarantees family=A region=d9a7fdf8b82889b95543ffed36720442] 2024-11-08T00:24:27,332 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108b916035a867d4a9eacf8cd84ab62a5e9_d9a7fdf8b82889b95543ffed36720442 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d9a7fdf8b82889b95543ffed36720442] 2024-11-08T00:24:27,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742012_1188 (size=12949) 2024-11-08T00:24:27,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742013_1189 (size=4469) 2024-11-08T00:24:27,341 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9a7fdf8b82889b95543ffed36720442#A#compaction#153 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:27,341 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/d2a954a055c646ca843115206ba785e4 is 175, key is test_row_0/A:col10/1731025465222/Put/seqid=0 2024-11-08T00:24:27,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-08T00:24:27,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:27,346 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing d9a7fdf8b82889b95543ffed36720442 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-08T00:24:27,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742014_1190 (size=31903) 2024-11-08T00:24:27,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108fca2de3627b847e7986357943e43b554_d9a7fdf8b82889b95543ffed36720442 is 50, key is test_row_0/A:col10/1731025466393/Put/seqid=0 2024-11-08T00:24:27,366 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/d2a954a055c646ca843115206ba785e4 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/d2a954a055c646ca843115206ba785e4 2024-11-08T00:24:27,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,377 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9a7fdf8b82889b95543ffed36720442/A of d9a7fdf8b82889b95543ffed36720442 into d2a954a055c646ca843115206ba785e4(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:27,377 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9a7fdf8b82889b95543ffed36720442: 2024-11-08T00:24:27,377 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., storeName=d9a7fdf8b82889b95543ffed36720442/A, priority=13, startTime=1731025467296; duration=0sec 2024-11-08T00:24:27,377 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:24:27,377 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9a7fdf8b82889b95543ffed36720442:A 2024-11-08T00:24:27,377 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:24:27,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,379 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 32363 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:24:27,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,379 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): d9a7fdf8b82889b95543ffed36720442/C is initiating minor compaction (all files) 2024-11-08T00:24:27,379 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9a7fdf8b82889b95543ffed36720442/C in TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:27,379 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/c57d5003ef924f06839ddc3d56dc18bf, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/e94e1d87625648cca8cb8f2d7c2d59f4, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/c34732d20a754ed89640f8bba02d0f2a] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp, totalSize=31.6 K 2024-11-08T00:24:27,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,380 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting c57d5003ef924f06839ddc3d56dc18bf, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731025462882 2024-11-08T00:24:27,381 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting e94e1d87625648cca8cb8f2d7c2d59f4, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731025464048 2024-11-08T00:24:27,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,382 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting c34732d20a754ed89640f8bba02d0f2a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731025465214 2024-11-08T00:24:27,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742015_1191 (size=9914) 2024-11-08T00:24:27,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:27,393 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108fca2de3627b847e7986357943e43b554_d9a7fdf8b82889b95543ffed36720442 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108fca2de3627b847e7986357943e43b554_d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:27,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/b6f1eb805ef94f008b8be5b28c42206e, store: [table=TestAcidGuarantees family=A region=d9a7fdf8b82889b95543ffed36720442] 2024-11-08T00:24:27,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/b6f1eb805ef94f008b8be5b28c42206e is 175, key is test_row_0/A:col10/1731025466393/Put/seqid=0 2024-11-08T00:24:27,396 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9a7fdf8b82889b95543ffed36720442#C#compaction#156 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:27,397 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/b5fde4c2413c4f728d600b2577d705e6 is 50, key is test_row_0/C:col10/1731025465222/Put/seqid=0 2024-11-08T00:24:27,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:27,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. as already flushing 2024-11-08T00:24:27,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742016_1192 (size=12949) 2024-11-08T00:24:27,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742017_1193 (size=22561) 2024-11-08T00:24:27,438 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=289, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/b6f1eb805ef94f008b8be5b28c42206e 2024-11-08T00:24:27,444 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/b5fde4c2413c4f728d600b2577d705e6 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/b5fde4c2413c4f728d600b2577d705e6 2024-11-08T00:24:27,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/4f762fdb4472441f898edd06d687528b is 50, key is test_row_0/B:col10/1731025466393/Put/seqid=0 2024-11-08T00:24:27,455 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9a7fdf8b82889b95543ffed36720442/C of d9a7fdf8b82889b95543ffed36720442 into b5fde4c2413c4f728d600b2577d705e6(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:27,455 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9a7fdf8b82889b95543ffed36720442: 2024-11-08T00:24:27,455 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., storeName=d9a7fdf8b82889b95543ffed36720442/C, priority=13, startTime=1731025467297; duration=0sec 2024-11-08T00:24:27,455 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:27,455 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9a7fdf8b82889b95543ffed36720442:C 2024-11-08T00:24:27,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742018_1194 (size=9857) 2024-11-08T00:24:27,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:27,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57324 deadline: 1731025527512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:27,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:27,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:27,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:27,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:27,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57290 deadline: 1731025527537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:27,540 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:27,540 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:27,540 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:27,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-08T00:24:27,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:27,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025527547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:27,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:27,548 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:27,548 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:27,548 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:27,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025527547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:27,548 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:27,548 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:27,548 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:27,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:27,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57324 deadline: 1731025527627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:27,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:27,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:27,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:27,740 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/0cf7fb5e11004dce9e3c6f4db1d2feb4 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/0cf7fb5e11004dce9e3c6f4db1d2feb4 2024-11-08T00:24:27,747 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9a7fdf8b82889b95543ffed36720442/B of d9a7fdf8b82889b95543ffed36720442 into 0cf7fb5e11004dce9e3c6f4db1d2feb4(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:27,747 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9a7fdf8b82889b95543ffed36720442: 2024-11-08T00:24:27,747 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., storeName=d9a7fdf8b82889b95543ffed36720442/B, priority=13, startTime=1731025467296; duration=0sec 2024-11-08T00:24:27,747 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:27,747 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9a7fdf8b82889b95543ffed36720442:B 2024-11-08T00:24:27,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:27,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57324 deadline: 1731025527837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:27,844 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:27,844 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:27,844 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:27,864 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/4f762fdb4472441f898edd06d687528b 2024-11-08T00:24:27,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/2820577ac23c4d93aa7eea77c2e54a4e is 50, key is test_row_0/C:col10/1731025466393/Put/seqid=0 2024-11-08T00:24:27,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742019_1195 (size=9857) 2024-11-08T00:24:28,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:28,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57324 deadline: 1731025528147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:28,154 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:28,154 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:28,154 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:28,307 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/2820577ac23c4d93aa7eea77c2e54a4e 2024-11-08T00:24:28,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/b6f1eb805ef94f008b8be5b28c42206e as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/b6f1eb805ef94f008b8be5b28c42206e 2024-11-08T00:24:28,333 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/b6f1eb805ef94f008b8be5b28c42206e, entries=100, sequenceid=289, filesize=22.0 K 2024-11-08T00:24:28,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/4f762fdb4472441f898edd06d687528b as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/4f762fdb4472441f898edd06d687528b 2024-11-08T00:24:28,344 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/4f762fdb4472441f898edd06d687528b, entries=100, sequenceid=289, filesize=9.6 K 2024-11-08T00:24:28,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/2820577ac23c4d93aa7eea77c2e54a4e as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/2820577ac23c4d93aa7eea77c2e54a4e 2024-11-08T00:24:28,353 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/2820577ac23c4d93aa7eea77c2e54a4e, entries=100, sequenceid=289, filesize=9.6 K 2024-11-08T00:24:28,354 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=181.14 KB/185490 for d9a7fdf8b82889b95543ffed36720442 in 1008ms, sequenceid=289, compaction requested=false 2024-11-08T00:24:28,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for d9a7fdf8b82889b95543ffed36720442: 2024-11-08T00:24:28,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:28,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-08T00:24:28,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-08T00:24:28,360 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-08T00:24:28,360 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9390 sec 2024-11-08T00:24:28,370 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.9520 sec 2024-11-08T00:24:28,550 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:24:28,551 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d9a7fdf8b82889b95543ffed36720442 3/3 column families, dataSize=187.85 KB heapSize=492.94 KB 2024-11-08T00:24:28,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:28,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-08T00:24:28,559 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:24:28,560 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-08T00:24:28,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-08T00:24:28,561 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108ba23c1f3297144ef9a3116b90d001b43_d9a7fdf8b82889b95543ffed36720442 is 50, key is test_row_0/A:col10/1731025467509/Put/seqid=0 2024-11-08T00:24:28,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-08T00:24:28,562 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T00:24:28,563 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T00:24:28,563 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T00:24:28,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:28,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57290 deadline: 1731025528560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:28,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:28,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:28,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:28,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742020_1196 (size=14994) 2024-11-08T00:24:28,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:28,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025528566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:28,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:28,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025528566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:28,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:28,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:28,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:28,570 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:28,570 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:28,570 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:28,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-08T00:24:28,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:28,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57324 deadline: 1731025528667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:28,668 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:28,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:28,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:28,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:28,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57290 deadline: 1731025528678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:28,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:28,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:28,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:28,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025528678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:28,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:28,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:28,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:28,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:28,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:28,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025528678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:28,679 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:28,679 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:28,679 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:28,716 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-08T00:24:28,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:28,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. as already flushing 2024-11-08T00:24:28,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:28,720 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:28,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:28,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:28,874 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-08T00:24:28,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:28,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. as already flushing 2024-11-08T00:24:28,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:28,874 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:28,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:28,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:28,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-08T00:24:28,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:28,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025528887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:28,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:28,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:28,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025528887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:28,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:28,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:28,891 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:28,891 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:28,891 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:28,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:28,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57290 deadline: 1731025528887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:28,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:28,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:28,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:28,971 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,001 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108ba23c1f3297144ef9a3116b90d001b43_d9a7fdf8b82889b95543ffed36720442 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108ba23c1f3297144ef9a3116b90d001b43_d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:29,002 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/850ec7e709124c67b0dbdb6bb02a7a47, store: [table=TestAcidGuarantees family=A region=d9a7fdf8b82889b95543ffed36720442] 2024-11-08T00:24:29,003 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/850ec7e709124c67b0dbdb6bb02a7a47 is 175, key is test_row_0/A:col10/1731025467509/Put/seqid=0 2024-11-08T00:24:29,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-08T00:24:29,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:29,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. as already flushing 2024-11-08T00:24:29,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:29,037 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:29,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:29,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:29,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742021_1197 (size=39949) 2024-11-08T00:24:29,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-08T00:24:29,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-08T00:24:29,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:29,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. as already flushing 2024-11-08T00:24:29,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:29,194 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:29,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:29,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:29,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:29,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025529197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:29,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:29,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025529197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:29,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:29,199 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:29,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:29,199 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:29,199 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:29,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:29,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:29,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57290 deadline: 1731025529207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:29,219 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:29,219 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:29,219 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:29,348 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-08T00:24:29,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:29,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. as already flushing 2024-11-08T00:24:29,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:29,348 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:29,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:29,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:29,440 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=323, memsize=62.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/850ec7e709124c67b0dbdb6bb02a7a47 2024-11-08T00:24:29,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/66118748fa5b404f86e9b766e8d23571 is 50, key is test_row_0/B:col10/1731025467509/Put/seqid=0 2024-11-08T00:24:29,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742022_1198 (size=12301) 2024-11-08T00:24:29,465 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/66118748fa5b404f86e9b766e8d23571 2024-11-08T00:24:29,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/cc18c26ec43e4c7b9073c54d4bf824c0 is 50, key is test_row_0/C:col10/1731025467509/Put/seqid=0 2024-11-08T00:24:29,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742023_1199 (size=12301) 2024-11-08T00:24:29,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-08T00:24:29,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:29,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. as already flushing 2024-11-08T00:24:29,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:29,502 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:29,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:29,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:29,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-08T00:24:29,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:29,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. as already flushing 2024-11-08T00:24:29,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:29,656 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:29,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:29,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:29,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:29,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57324 deadline: 1731025529688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:29,691 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 3302f0f507bd,40479,1731025415856 for 'test_row_1' in d9a7fdf8b82889b95543ffed36720442 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T00:24:29,692 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:29,692 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:29,692 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:29,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-08T00:24:29,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:29,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025529707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:29,710 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:29,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:29,710 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:29,710 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:29,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025529707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:29,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:29,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:29,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:29,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:29,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57290 deadline: 1731025529727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:29,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:29,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:29,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:29,814 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-08T00:24:29,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:29,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. as already flushing 2024-11-08T00:24:29,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:29,814 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:29,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:29,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:29,897 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/cc18c26ec43e4c7b9073c54d4bf824c0 2024-11-08T00:24:29,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/850ec7e709124c67b0dbdb6bb02a7a47 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/850ec7e709124c67b0dbdb6bb02a7a47 2024-11-08T00:24:29,910 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/850ec7e709124c67b0dbdb6bb02a7a47, entries=200, sequenceid=323, filesize=39.0 K 2024-11-08T00:24:29,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/66118748fa5b404f86e9b766e8d23571 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/66118748fa5b404f86e9b766e8d23571 2024-11-08T00:24:29,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/66118748fa5b404f86e9b766e8d23571, entries=150, sequenceid=323, filesize=12.0 K 2024-11-08T00:24:29,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/cc18c26ec43e4c7b9073c54d4bf824c0 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/cc18c26ec43e4c7b9073c54d4bf824c0 2024-11-08T00:24:29,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,930 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/cc18c26ec43e4c7b9073c54d4bf824c0, entries=150, sequenceid=323, filesize=12.0 K 2024-11-08T00:24:29,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,933 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~187.85 KB/192360, heapSize ~492.89 KB/504720, currentSize=20.13 KB/20610 for d9a7fdf8b82889b95543ffed36720442 in 1382ms, sequenceid=323, compaction requested=true 2024-11-08T00:24:29,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d9a7fdf8b82889b95543ffed36720442: 2024-11-08T00:24:29,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9a7fdf8b82889b95543ffed36720442:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T00:24:29,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:29,933 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:24:29,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9a7fdf8b82889b95543ffed36720442:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T00:24:29,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:24:29,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9a7fdf8b82889b95543ffed36720442:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:24:29,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:24:29,933 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:24:29,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,939 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:24:29,939 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:24:29,939 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): d9a7fdf8b82889b95543ffed36720442/B is initiating minor compaction (all files) 2024-11-08T00:24:29,939 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): d9a7fdf8b82889b95543ffed36720442/A is initiating minor compaction (all files) 2024-11-08T00:24:29,939 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9a7fdf8b82889b95543ffed36720442/A in TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:29,939 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9a7fdf8b82889b95543ffed36720442/B in TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:29,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,939 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/0cf7fb5e11004dce9e3c6f4db1d2feb4, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/4f762fdb4472441f898edd06d687528b, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/66118748fa5b404f86e9b766e8d23571] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp, totalSize=34.3 K 2024-11-08T00:24:29,939 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/d2a954a055c646ca843115206ba785e4, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/b6f1eb805ef94f008b8be5b28c42206e, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/850ec7e709124c67b0dbdb6bb02a7a47] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp, totalSize=92.2 K 2024-11-08T00:24:29,940 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:29,940 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0cf7fb5e11004dce9e3c6f4db1d2feb4, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731025465214 2024-11-08T00:24:29,940 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. files: [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/d2a954a055c646ca843115206ba785e4, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/b6f1eb805ef94f008b8be5b28c42206e, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/850ec7e709124c67b0dbdb6bb02a7a47] 2024-11-08T00:24:29,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,940 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting d2a954a055c646ca843115206ba785e4, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731025465214 2024-11-08T00:24:29,940 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f762fdb4472441f898edd06d687528b, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731025466393 2024-11-08T00:24:29,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,941 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting b6f1eb805ef94f008b8be5b28c42206e, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731025466393 2024-11-08T00:24:29,941 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66118748fa5b404f86e9b766e8d23571, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731025467507 2024-11-08T00:24:29,941 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 850ec7e709124c67b0dbdb6bb02a7a47, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731025467507 2024-11-08T00:24:29,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,972 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9a7fdf8b82889b95543ffed36720442#B#compaction#162 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:29,972 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/a6573948946f43619a94c22f773fd1e6 is 50, key is test_row_0/B:col10/1731025467509/Put/seqid=0 2024-11-08T00:24:29,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,973 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-08T00:24:29,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:29,974 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing d9a7fdf8b82889b95543ffed36720442 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-08T00:24:29,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,985 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d9a7fdf8b82889b95543ffed36720442] 2024-11-08T00:24:29,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:29,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411082e94e9ee0a3841109bdccaac621cec2b_d9a7fdf8b82889b95543ffed36720442 is 50, key is test_row_0/A:col10/1731025468556/Put/seqid=0 2024-11-08T00:24:30,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,034 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241108f6c264dc84774ce79103960e23464c60_d9a7fdf8b82889b95543ffed36720442 store=[table=TestAcidGuarantees family=A region=d9a7fdf8b82889b95543ffed36720442] 2024-11-08T00:24:30,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,036 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241108f6c264dc84774ce79103960e23464c60_d9a7fdf8b82889b95543ffed36720442, store=[table=TestAcidGuarantees family=A region=d9a7fdf8b82889b95543ffed36720442] 2024-11-08T00:24:30,037 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108f6c264dc84774ce79103960e23464c60_d9a7fdf8b82889b95543ffed36720442 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d9a7fdf8b82889b95543ffed36720442] 2024-11-08T00:24:30,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742024_1200 (size=13051) 2024-11-08T00:24:30,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,094 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/a6573948946f43619a94c22f773fd1e6 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/a6573948946f43619a94c22f773fd1e6 2024-11-08T00:24:30,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742026_1202 (size=4469) 2024-11-08T00:24:30,114 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9a7fdf8b82889b95543ffed36720442#A#compaction#163 average throughput is 0.19 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:30,115 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/96734715b60e47149498d50c0b110a88 is 175, key is test_row_0/A:col10/1731025467509/Put/seqid=0 2024-11-08T00:24:30,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742025_1201 (size=9914) 2024-11-08T00:24:30,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,136 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9a7fdf8b82889b95543ffed36720442/B of d9a7fdf8b82889b95543ffed36720442 into a6573948946f43619a94c22f773fd1e6(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:30,136 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9a7fdf8b82889b95543ffed36720442: 2024-11-08T00:24:30,136 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., storeName=d9a7fdf8b82889b95543ffed36720442/B, priority=13, startTime=1731025469933; duration=0sec 2024-11-08T00:24:30,136 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:24:30,136 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9a7fdf8b82889b95543ffed36720442:B 2024-11-08T00:24:30,136 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:24:30,141 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:24:30,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,141 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): d9a7fdf8b82889b95543ffed36720442/C is initiating minor compaction (all files) 2024-11-08T00:24:30,141 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9a7fdf8b82889b95543ffed36720442/C in TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:30,141 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/b5fde4c2413c4f728d600b2577d705e6, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/2820577ac23c4d93aa7eea77c2e54a4e, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/cc18c26ec43e4c7b9073c54d4bf824c0] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp, totalSize=34.3 K 2024-11-08T00:24:30,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,143 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5fde4c2413c4f728d600b2577d705e6, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731025465214 2024-11-08T00:24:30,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,152 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2820577ac23c4d93aa7eea77c2e54a4e, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731025466393 2024-11-08T00:24:30,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,154 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc18c26ec43e4c7b9073c54d4bf824c0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731025467507 2024-11-08T00:24:30,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742027_1203 (size=32005) 2024-11-08T00:24:30,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,218 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9a7fdf8b82889b95543ffed36720442#C#compaction#165 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:30,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,219 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/8f83287868464ef189c3bd65a2e33257 is 50, key is test_row_0/C:col10/1731025467509/Put/seqid=0 2024-11-08T00:24:30,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742028_1204 (size=13051) 2024-11-08T00:24:30,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,539 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411082e94e9ee0a3841109bdccaac621cec2b_d9a7fdf8b82889b95543ffed36720442 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411082e94e9ee0a3841109bdccaac621cec2b_d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:30,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/110645565ca74fe5bcb7bfb57e2ba990, store: [table=TestAcidGuarantees family=A region=d9a7fdf8b82889b95543ffed36720442] 2024-11-08T00:24:30,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/110645565ca74fe5bcb7bfb57e2ba990 is 175, key is test_row_0/A:col10/1731025468556/Put/seqid=0 2024-11-08T00:24:30,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742029_1205 (size=22561) 2024-11-08T00:24:30,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,610 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/96734715b60e47149498d50c0b110a88 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/96734715b60e47149498d50c0b110a88 2024-11-08T00:24:30,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,619 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9a7fdf8b82889b95543ffed36720442/A of d9a7fdf8b82889b95543ffed36720442 into 96734715b60e47149498d50c0b110a88(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:30,619 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9a7fdf8b82889b95543ffed36720442: 2024-11-08T00:24:30,619 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., storeName=d9a7fdf8b82889b95543ffed36720442/A, priority=13, startTime=1731025469933; duration=0sec 2024-11-08T00:24:30,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,620 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:30,620 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9a7fdf8b82889b95543ffed36720442:A 2024-11-08T00:24:30,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,686 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/8f83287868464ef189c3bd65a2e33257 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/8f83287868464ef189c3bd65a2e33257 2024-11-08T00:24:30,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,708 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9a7fdf8b82889b95543ffed36720442/C of d9a7fdf8b82889b95543ffed36720442 into 8f83287868464ef189c3bd65a2e33257(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:30,708 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9a7fdf8b82889b95543ffed36720442: 2024-11-08T00:24:30,708 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., storeName=d9a7fdf8b82889b95543ffed36720442/C, priority=13, startTime=1731025469933; duration=0sec 2024-11-08T00:24:30,708 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:30,708 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9a7fdf8b82889b95543ffed36720442:C 2024-11-08T00:24:30,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-08T00:24:30,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:30,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:30,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. as already flushing 2024-11-08T00:24:30,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:30,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025530828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:30,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:30,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:30,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:30,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:30,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025530835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:30,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:30,843 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:30,843 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:30,843 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:30,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57290 deadline: 1731025530839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:30,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:30,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:30,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:30,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:30,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025530948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:30,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:30,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025530948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:30,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:30,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:30,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:30,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:30,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57290 deadline: 1731025530948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:30,951 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:30,951 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:30,951 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:30,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:30,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:30,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:30,985 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=329, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/110645565ca74fe5bcb7bfb57e2ba990 2024-11-08T00:24:31,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/a13744b89cae4b328dc31920c86b7494 is 50, key is test_row_0/B:col10/1731025468556/Put/seqid=0 2024-11-08T00:24:31,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742030_1206 (size=9857) 2024-11-08T00:24:31,012 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/a13744b89cae4b328dc31920c86b7494 2024-11-08T00:24:31,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/03b557cfc72248c5ae5844ced3c4df30 is 50, key is test_row_0/C:col10/1731025468556/Put/seqid=0 2024-11-08T00:24:31,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742031_1207 (size=9857) 2024-11-08T00:24:31,076 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/03b557cfc72248c5ae5844ced3c4df30 2024-11-08T00:24:31,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/110645565ca74fe5bcb7bfb57e2ba990 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/110645565ca74fe5bcb7bfb57e2ba990 2024-11-08T00:24:31,101 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/110645565ca74fe5bcb7bfb57e2ba990, entries=100, sequenceid=329, filesize=22.0 K 2024-11-08T00:24:31,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/a13744b89cae4b328dc31920c86b7494 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/a13744b89cae4b328dc31920c86b7494 2024-11-08T00:24:31,119 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/a13744b89cae4b328dc31920c86b7494, entries=100, sequenceid=329, filesize=9.6 K 2024-11-08T00:24:31,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/03b557cfc72248c5ae5844ced3c4df30 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/03b557cfc72248c5ae5844ced3c4df30 2024-11-08T00:24:31,126 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/03b557cfc72248c5ae5844ced3c4df30, entries=100, sequenceid=329, filesize=9.6 K 2024-11-08T00:24:31,140 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=181.14 KB/185490 for d9a7fdf8b82889b95543ffed36720442 in 1166ms, sequenceid=329, compaction requested=false 2024-11-08T00:24:31,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for d9a7fdf8b82889b95543ffed36720442: 2024-11-08T00:24:31,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:31,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-08T00:24:31,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-08T00:24:31,149 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-08T00:24:31,149 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5840 sec 2024-11-08T00:24:31,151 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 2.5900 sec 2024-11-08T00:24:31,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:31,168 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:24:31,168 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d9a7fdf8b82889b95543ffed36720442 3/3 column families, dataSize=187.85 KB heapSize=492.94 KB 2024-11-08T00:24:31,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:31,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57290 deadline: 1731025531174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:31,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:31,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:31,181 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:31,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:31,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025531174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:31,181 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:31,181 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:31,181 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:31,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:31,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025531179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:31,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:31,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:31,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:31,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110869009e6a7d364b658ce0a50ede3511df_d9a7fdf8b82889b95543ffed36720442 is 50, key is test_row_0/A:col10/1731025470834/Put/seqid=0 2024-11-08T00:24:31,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742032_1208 (size=17534) 2024-11-08T00:24:31,228 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:31,237 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110869009e6a7d364b658ce0a50ede3511df_d9a7fdf8b82889b95543ffed36720442 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110869009e6a7d364b658ce0a50ede3511df_d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:31,239 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/9b1c33033f81419096bac1a7b035e0aa, store: [table=TestAcidGuarantees family=A region=d9a7fdf8b82889b95543ffed36720442] 2024-11-08T00:24:31,240 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/9b1c33033f81419096bac1a7b035e0aa is 175, key is test_row_0/A:col10/1731025470834/Put/seqid=0 2024-11-08T00:24:31,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742033_1209 (size=48639) 2024-11-08T00:24:31,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:31,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025531288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:31,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:31,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57290 deadline: 1731025531288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:31,289 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:31,289 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:31,289 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:31,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:31,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025531288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:31,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:31,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:31,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:31,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:31,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:31,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:31,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:31,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025531497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:31,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:31,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:31,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57290 deadline: 1731025531497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:31,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:31,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:31,501 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:31,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:31,501 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:31,501 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:31,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025531497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:31,501 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:31,501 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:31,501 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:31,685 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=365, memsize=67.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/9b1c33033f81419096bac1a7b035e0aa 2024-11-08T00:24:31,693 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/93c8483be8f04b228e0cdbac0bea73c5 is 50, key is test_row_0/B:col10/1731025470834/Put/seqid=0 2024-11-08T00:24:31,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742034_1210 (size=12301) 2024-11-08T00:24:31,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:31,698 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.09 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/93c8483be8f04b228e0cdbac0bea73c5 2024-11-08T00:24:31,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57324 deadline: 1731025531697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:31,698 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 3302f0f507bd,40479,1731025415856 for 'test_row_1' in d9a7fdf8b82889b95543ffed36720442 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T00:24:31,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:31,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:31,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:31,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/16b664ecf04b4978bcf04a8da60f3156 is 50, key is test_row_0/C:col10/1731025470834/Put/seqid=0 2024-11-08T00:24:31,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742035_1211 (size=12301) 2024-11-08T00:24:31,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:31,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025531807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:31,808 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:31,808 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:31,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:31,808 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:31,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025531807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:31,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:31,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:31,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:31,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:31,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57290 deadline: 1731025531809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:31,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:31,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:31,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:32,111 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.09 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/16b664ecf04b4978bcf04a8da60f3156 2024-11-08T00:24:32,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/9b1c33033f81419096bac1a7b035e0aa as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/9b1c33033f81419096bac1a7b035e0aa 2024-11-08T00:24:32,120 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/9b1c33033f81419096bac1a7b035e0aa, entries=250, sequenceid=365, filesize=47.5 K 2024-11-08T00:24:32,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/93c8483be8f04b228e0cdbac0bea73c5 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/93c8483be8f04b228e0cdbac0bea73c5 2024-11-08T00:24:32,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,131 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/93c8483be8f04b228e0cdbac0bea73c5, entries=150, sequenceid=365, filesize=12.0 K 2024-11-08T00:24:32,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/16b664ecf04b4978bcf04a8da60f3156 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/16b664ecf04b4978bcf04a8da60f3156 2024-11-08T00:24:32,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,141 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/16b664ecf04b4978bcf04a8da60f3156, entries=150, sequenceid=365, filesize=12.0 K 2024-11-08T00:24:32,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,143 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~201.27 KB/206100, heapSize ~528.05 KB/540720, currentSize=0 B/0 for d9a7fdf8b82889b95543ffed36720442 in 975ms, sequenceid=365, compaction requested=true 2024-11-08T00:24:32,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d9a7fdf8b82889b95543ffed36720442: 2024-11-08T00:24:32,143 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:24:32,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9a7fdf8b82889b95543ffed36720442:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T00:24:32,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:32,144 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:24:32,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9a7fdf8b82889b95543ffed36720442:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T00:24:32,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:32,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9a7fdf8b82889b95543ffed36720442:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:24:32,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:24:32,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,145 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103205 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:24:32,145 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): d9a7fdf8b82889b95543ffed36720442/A is initiating minor compaction (all files) 2024-11-08T00:24:32,145 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9a7fdf8b82889b95543ffed36720442/A in TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:32,145 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/96734715b60e47149498d50c0b110a88, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/110645565ca74fe5bcb7bfb57e2ba990, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/9b1c33033f81419096bac1a7b035e0aa] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp, totalSize=100.8 K 2024-11-08T00:24:32,145 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:32,145 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. files: [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/96734715b60e47149498d50c0b110a88, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/110645565ca74fe5bcb7bfb57e2ba990, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/9b1c33033f81419096bac1a7b035e0aa] 2024-11-08T00:24:32,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,146 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:24:32,146 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): d9a7fdf8b82889b95543ffed36720442/B is initiating minor compaction (all files) 2024-11-08T00:24:32,146 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9a7fdf8b82889b95543ffed36720442/B in TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:32,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,146 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/a6573948946f43619a94c22f773fd1e6, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/a13744b89cae4b328dc31920c86b7494, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/93c8483be8f04b228e0cdbac0bea73c5] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp, totalSize=34.4 K 2024-11-08T00:24:32,146 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96734715b60e47149498d50c0b110a88, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731025467507 2024-11-08T00:24:32,147 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting a6573948946f43619a94c22f773fd1e6, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731025467507 2024-11-08T00:24:32,147 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 110645565ca74fe5bcb7bfb57e2ba990, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731025468556 2024-11-08T00:24:32,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,147 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting a13744b89cae4b328dc31920c86b7494, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731025468556 2024-11-08T00:24:32,147 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b1c33033f81419096bac1a7b035e0aa, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731025470826 2024-11-08T00:24:32,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,148 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 93c8483be8f04b228e0cdbac0bea73c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731025470834 2024-11-08T00:24:32,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,161 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d9a7fdf8b82889b95543ffed36720442] 2024-11-08T00:24:32,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,169 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9a7fdf8b82889b95543ffed36720442#B#compaction#172 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:32,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,170 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/d7cf8e26514b420385ace8b04adc3cc2 is 50, key is test_row_0/B:col10/1731025470834/Put/seqid=0 2024-11-08T00:24:32,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,171 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110854d6fc30c1f248eb8f5a00011a3ecaf7_d9a7fdf8b82889b95543ffed36720442 store=[table=TestAcidGuarantees family=A region=d9a7fdf8b82889b95543ffed36720442] 2024-11-08T00:24:32,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,177 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110854d6fc30c1f248eb8f5a00011a3ecaf7_d9a7fdf8b82889b95543ffed36720442, store=[table=TestAcidGuarantees family=A region=d9a7fdf8b82889b95543ffed36720442] 2024-11-08T00:24:32,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,177 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110854d6fc30c1f248eb8f5a00011a3ecaf7_d9a7fdf8b82889b95543ffed36720442 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d9a7fdf8b82889b95543ffed36720442] 2024-11-08T00:24:32,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742036_1212 (size=13153) 2024-11-08T00:24:32,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742037_1213 (size=4469) 2024-11-08T00:24:32,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,219 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] io.ByteBuffAllocator(384): Pool already reached its max capacity : 186 and no free buffers now. Consider increasing the value for 'hbase.server.allocator.max.buffer.count' ? 2024-11-08T00:24:32,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:32,474 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:24:32,474 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d9a7fdf8b82889b95543ffed36720442 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T00:24:32,509 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108e388e45138c34a93b758222b262c2eb6_d9a7fdf8b82889b95543ffed36720442 is 50, key is test_row_0/A:col10/1731025472424/Put/seqid=0 2024-11-08T00:24:32,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742038_1214 (size=20074) 2024-11-08T00:24:32,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:32,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57290 deadline: 1731025532603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:32,609 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:32,609 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:32,609 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:32,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:32,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025532605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:32,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:32,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025532606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:32,611 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:32,611 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:32,611 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:32,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:32,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:32,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:32,612 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9a7fdf8b82889b95543ffed36720442#A#compaction#171 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:32,613 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/1e4720efda4740df89b8f4ef5f0c334e is 175, key is test_row_0/A:col10/1731025470834/Put/seqid=0 2024-11-08T00:24:32,617 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/d7cf8e26514b420385ace8b04adc3cc2 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/d7cf8e26514b420385ace8b04adc3cc2 2024-11-08T00:24:32,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742039_1215 (size=32107) 2024-11-08T00:24:32,638 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9a7fdf8b82889b95543ffed36720442/B of d9a7fdf8b82889b95543ffed36720442 into d7cf8e26514b420385ace8b04adc3cc2(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:32,638 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9a7fdf8b82889b95543ffed36720442: 2024-11-08T00:24:32,638 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., storeName=d9a7fdf8b82889b95543ffed36720442/B, priority=13, startTime=1731025472144; duration=0sec 2024-11-08T00:24:32,639 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:24:32,639 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9a7fdf8b82889b95543ffed36720442:B 2024-11-08T00:24:32,639 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:24:32,641 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:24:32,641 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): d9a7fdf8b82889b95543ffed36720442/C is initiating minor compaction (all files) 2024-11-08T00:24:32,641 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9a7fdf8b82889b95543ffed36720442/C in TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:32,641 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/8f83287868464ef189c3bd65a2e33257, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/03b557cfc72248c5ae5844ced3c4df30, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/16b664ecf04b4978bcf04a8da60f3156] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp, totalSize=34.4 K 2024-11-08T00:24:32,642 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f83287868464ef189c3bd65a2e33257, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731025467507 2024-11-08T00:24:32,643 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 03b557cfc72248c5ae5844ced3c4df30, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731025468556 2024-11-08T00:24:32,643 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/1e4720efda4740df89b8f4ef5f0c334e as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/1e4720efda4740df89b8f4ef5f0c334e 2024-11-08T00:24:32,644 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 16b664ecf04b4978bcf04a8da60f3156, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731025470834 2024-11-08T00:24:32,651 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9a7fdf8b82889b95543ffed36720442/A of d9a7fdf8b82889b95543ffed36720442 into 1e4720efda4740df89b8f4ef5f0c334e(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:32,651 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9a7fdf8b82889b95543ffed36720442: 2024-11-08T00:24:32,651 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., storeName=d9a7fdf8b82889b95543ffed36720442/A, priority=13, startTime=1731025472143; duration=0sec 2024-11-08T00:24:32,651 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:32,652 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9a7fdf8b82889b95543ffed36720442:A 2024-11-08T00:24:32,662 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9a7fdf8b82889b95543ffed36720442#C#compaction#174 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:32,663 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/23fcb08a68f04acc890e87b6f5e74fc6 is 50, key is test_row_0/C:col10/1731025470834/Put/seqid=0 2024-11-08T00:24:32,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742040_1216 (size=13153) 2024-11-08T00:24:32,688 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/23fcb08a68f04acc890e87b6f5e74fc6 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/23fcb08a68f04acc890e87b6f5e74fc6 2024-11-08T00:24:32,693 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9a7fdf8b82889b95543ffed36720442/C of d9a7fdf8b82889b95543ffed36720442 into 23fcb08a68f04acc890e87b6f5e74fc6(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:32,693 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9a7fdf8b82889b95543ffed36720442: 2024-11-08T00:24:32,693 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., storeName=d9a7fdf8b82889b95543ffed36720442/C, priority=13, startTime=1731025472144; duration=0sec 2024-11-08T00:24:32,693 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:32,693 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9a7fdf8b82889b95543ffed36720442:C 2024-11-08T00:24:32,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:32,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57290 deadline: 1731025532717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:32,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:32,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:32,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:32,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:32,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025532717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:32,721 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:32,721 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:32,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:32,721 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:32,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025532717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:32,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:32,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:32,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:32,726 INFO [Thread-587 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-587. 2024-11-08T00:24:32,726 DEBUG [Thread-587 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T00:24:32,726 DEBUG [Thread-587 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:24:32,726 DEBUG [Thread-587 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:24:32,726 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:24:32,727 INFO [Thread-585 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-585. 2024-11-08T00:24:32,727 DEBUG [Thread-585 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T00:24:32,727 DEBUG [Thread-585 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:24:32,727 DEBUG [Thread-585 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:24:32,727 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:24:32,732 INFO [Thread-591 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-591. 2024-11-08T00:24:32,732 DEBUG [Thread-591 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T00:24:32,732 DEBUG [Thread-591 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:24:32,732 DEBUG [Thread-591 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:24:32,732 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:24:32,735 INFO [Thread-589 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-589. 2024-11-08T00:24:32,736 DEBUG [Thread-589 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T00:24:32,736 DEBUG [Thread-589 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:24:32,736 DEBUG [Thread-589 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:24:32,736 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:24:32,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-08T00:24:32,738 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:24:32,918 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T00:24:32,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:32,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:32,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:32,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025532927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:32,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025532927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:32,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57290 deadline: 1731025532927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:32,928 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:32,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:32,928 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:32,928 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:32,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:32,928 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:32,928 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:32,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:32,928 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:32,964 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:32,967 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108e388e45138c34a93b758222b262c2eb6_d9a7fdf8b82889b95543ffed36720442 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108e388e45138c34a93b758222b262c2eb6_d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:32,968 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/c1c8c0ad629e46bfa2cb7a284785f572, store: [table=TestAcidGuarantees family=A region=d9a7fdf8b82889b95543ffed36720442] 2024-11-08T00:24:32,969 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/c1c8c0ad629e46bfa2cb7a284785f572 is 175, key is test_row_0/A:col10/1731025472424/Put/seqid=0 2024-11-08T00:24:32,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742041_1217 (size=57333) 2024-11-08T00:24:33,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:33,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57290 deadline: 1731025533237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:33,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:33,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025533237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:33,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:33,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:33,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:33,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:33,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025533237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:33,238 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:33,238 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:33,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:33,238 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:33,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:33,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:33,373 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=376, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/c1c8c0ad629e46bfa2cb7a284785f572 2024-11-08T00:24:33,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/ade81c1c31aa41d08070e0c8b65e6ab8 is 50, key is test_row_0/B:col10/1731025472424/Put/seqid=0 2024-11-08T00:24:33,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742042_1218 (size=12301) 2024-11-08T00:24:33,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:33,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:33,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:33,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57332 deadline: 1731025533748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:33,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57290 deadline: 1731025533748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:33,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57288 deadline: 1731025533748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:33,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:33,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:33,748 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:33,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:33,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:33,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:33,748 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:33,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:33,749 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:24:33,784 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/ade81c1c31aa41d08070e0c8b65e6ab8 2024-11-08T00:24:33,790 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/1d512628e2c14f19ab7a6ac5cdd7adde is 50, key is test_row_0/C:col10/1731025472424/Put/seqid=0 2024-11-08T00:24:33,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742043_1219 (size=12301) 2024-11-08T00:24:34,194 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/1d512628e2c14f19ab7a6ac5cdd7adde 2024-11-08T00:24:34,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/c1c8c0ad629e46bfa2cb7a284785f572 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/c1c8c0ad629e46bfa2cb7a284785f572 2024-11-08T00:24:34,202 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/c1c8c0ad629e46bfa2cb7a284785f572, entries=300, sequenceid=376, filesize=56.0 K 2024-11-08T00:24:34,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/ade81c1c31aa41d08070e0c8b65e6ab8 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/ade81c1c31aa41d08070e0c8b65e6ab8 2024-11-08T00:24:34,207 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/ade81c1c31aa41d08070e0c8b65e6ab8, entries=150, sequenceid=376, filesize=12.0 K 2024-11-08T00:24:34,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/1d512628e2c14f19ab7a6ac5cdd7adde as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/1d512628e2c14f19ab7a6ac5cdd7adde 2024-11-08T00:24:34,212 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/1d512628e2c14f19ab7a6ac5cdd7adde, entries=150, sequenceid=376, filesize=12.0 K 2024-11-08T00:24:34,213 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for d9a7fdf8b82889b95543ffed36720442 in 1739ms, sequenceid=376, compaction requested=false 2024-11-08T00:24:34,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d9a7fdf8b82889b95543ffed36720442: 2024-11-08T00:24:34,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:34,759 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:24:34,759 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d9a7fdf8b82889b95543ffed36720442 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-08T00:24:34,759 INFO [Thread-580 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-580. 2024-11-08T00:24:34,759 INFO [Thread-574 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-574. 2024-11-08T00:24:34,759 DEBUG [Thread-574 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T00:24:34,759 DEBUG [Thread-580 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T00:24:34,759 DEBUG [Thread-574 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:24:34,759 DEBUG [Thread-580 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:24:34,759 DEBUG [Thread-580 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:24:34,759 DEBUG [Thread-574 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:24:34,759 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:24:34,759 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:24:34,766 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411083b3e57e15bd444068326a6e394f8cabb_d9a7fdf8b82889b95543ffed36720442 is 50, key is test_row_0/A:col10/1731025472593/Put/seqid=0 2024-11-08T00:24:34,768 INFO [Thread-576 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-576. 2024-11-08T00:24:34,768 DEBUG [Thread-576 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T00:24:34,768 DEBUG [Thread-576 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:24:34,768 DEBUG [Thread-576 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:24:34,768 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:24:34,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742044_1220 (size=12454) 2024-11-08T00:24:35,171 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:35,175 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411083b3e57e15bd444068326a6e394f8cabb_d9a7fdf8b82889b95543ffed36720442 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411083b3e57e15bd444068326a6e394f8cabb_d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:35,176 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/f363505d76b74ed2b0eda78e3d342a5a, store: [table=TestAcidGuarantees family=A region=d9a7fdf8b82889b95543ffed36720442] 2024-11-08T00:24:35,176 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/f363505d76b74ed2b0eda78e3d342a5a is 175, key is test_row_0/A:col10/1731025472593/Put/seqid=0 2024-11-08T00:24:35,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742045_1221 (size=31255) 2024-11-08T00:24:35,580 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=406, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/f363505d76b74ed2b0eda78e3d342a5a 2024-11-08T00:24:35,589 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/da0740e2e6b447a296856582bb484880 is 50, key is test_row_0/B:col10/1731025472593/Put/seqid=0 2024-11-08T00:24:35,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742046_1222 (size=12301) 2024-11-08T00:24:35,739 INFO [Thread-582 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-582. 2024-11-08T00:24:35,739 DEBUG [Thread-582 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T00:24:35,739 DEBUG [Thread-582 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:24:35,739 DEBUG [Thread-582 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:24:35,739 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:24:35,994 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/da0740e2e6b447a296856582bb484880 2024-11-08T00:24:36,001 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/8aac8454126f4a71b7b9a20d841e87ff is 50, key is test_row_0/C:col10/1731025472593/Put/seqid=0 2024-11-08T00:24:36,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742047_1223 (size=12301) 2024-11-08T00:24:36,008 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/8aac8454126f4a71b7b9a20d841e87ff 2024-11-08T00:24:36,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/f363505d76b74ed2b0eda78e3d342a5a as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/f363505d76b74ed2b0eda78e3d342a5a 2024-11-08T00:24:36,018 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/f363505d76b74ed2b0eda78e3d342a5a, entries=150, sequenceid=406, filesize=30.5 K 2024-11-08T00:24:36,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/da0740e2e6b447a296856582bb484880 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/da0740e2e6b447a296856582bb484880 2024-11-08T00:24:36,025 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/da0740e2e6b447a296856582bb484880, entries=150, sequenceid=406, filesize=12.0 K 2024-11-08T00:24:36,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/8aac8454126f4a71b7b9a20d841e87ff as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/8aac8454126f4a71b7b9a20d841e87ff 2024-11-08T00:24:36,034 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/8aac8454126f4a71b7b9a20d841e87ff, entries=150, sequenceid=406, filesize=12.0 K 2024-11-08T00:24:36,035 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=13.42 KB/13740 for d9a7fdf8b82889b95543ffed36720442 in 1276ms, sequenceid=406, compaction requested=true 2024-11-08T00:24:36,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d9a7fdf8b82889b95543ffed36720442: 2024-11-08T00:24:36,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9a7fdf8b82889b95543ffed36720442:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T00:24:36,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:36,035 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:24:36,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9a7fdf8b82889b95543ffed36720442:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T00:24:36,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:36,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d9a7fdf8b82889b95543ffed36720442:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:24:36,035 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:24:36,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:24:36,036 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:24:36,036 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 120695 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:24:36,037 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): d9a7fdf8b82889b95543ffed36720442/A is initiating minor compaction (all files) 2024-11-08T00:24:36,037 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): d9a7fdf8b82889b95543ffed36720442/B is initiating minor compaction (all files) 2024-11-08T00:24:36,037 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9a7fdf8b82889b95543ffed36720442/B in TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:36,037 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9a7fdf8b82889b95543ffed36720442/A in TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:36,037 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/d7cf8e26514b420385ace8b04adc3cc2, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/ade81c1c31aa41d08070e0c8b65e6ab8, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/da0740e2e6b447a296856582bb484880] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp, totalSize=36.9 K 2024-11-08T00:24:36,037 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/1e4720efda4740df89b8f4ef5f0c334e, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/c1c8c0ad629e46bfa2cb7a284785f572, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/f363505d76b74ed2b0eda78e3d342a5a] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp, totalSize=117.9 K 2024-11-08T00:24:36,037 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:36,037 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. files: [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/1e4720efda4740df89b8f4ef5f0c334e, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/c1c8c0ad629e46bfa2cb7a284785f572, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/f363505d76b74ed2b0eda78e3d342a5a] 2024-11-08T00:24:36,037 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting d7cf8e26514b420385ace8b04adc3cc2, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731025470834 2024-11-08T00:24:36,037 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e4720efda4740df89b8f4ef5f0c334e, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731025470834 2024-11-08T00:24:36,038 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1c8c0ad629e46bfa2cb7a284785f572, keycount=300, bloomtype=ROW, size=56.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731025472419 2024-11-08T00:24:36,038 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting ade81c1c31aa41d08070e0c8b65e6ab8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731025472419 2024-11-08T00:24:36,038 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting f363505d76b74ed2b0eda78e3d342a5a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1731025472593 2024-11-08T00:24:36,038 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting da0740e2e6b447a296856582bb484880, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1731025472593 2024-11-08T00:24:36,051 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9a7fdf8b82889b95543ffed36720442#B#compaction#180 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:36,052 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/2e7afa8fd5be4a558f0dedc2127cdf01 is 50, key is test_row_0/B:col10/1731025472593/Put/seqid=0 2024-11-08T00:24:36,053 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d9a7fdf8b82889b95543ffed36720442] 2024-11-08T00:24:36,056 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411085271b777f7cd4b11a8f1eda3f22f28ff_d9a7fdf8b82889b95543ffed36720442 store=[table=TestAcidGuarantees family=A region=d9a7fdf8b82889b95543ffed36720442] 2024-11-08T00:24:36,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742048_1224 (size=13255) 2024-11-08T00:24:36,089 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411085271b777f7cd4b11a8f1eda3f22f28ff_d9a7fdf8b82889b95543ffed36720442, store=[table=TestAcidGuarantees family=A region=d9a7fdf8b82889b95543ffed36720442] 2024-11-08T00:24:36,089 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411085271b777f7cd4b11a8f1eda3f22f28ff_d9a7fdf8b82889b95543ffed36720442 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d9a7fdf8b82889b95543ffed36720442] 2024-11-08T00:24:36,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742049_1225 (size=4469) 2024-11-08T00:24:36,094 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9a7fdf8b82889b95543ffed36720442#A#compaction#181 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:36,094 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/8ffb35ea50ba4a13a56cb2e035c8d628 is 175, key is test_row_0/A:col10/1731025472593/Put/seqid=0 2024-11-08T00:24:36,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742050_1226 (size=32209) 2024-11-08T00:24:36,106 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/8ffb35ea50ba4a13a56cb2e035c8d628 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/8ffb35ea50ba4a13a56cb2e035c8d628 2024-11-08T00:24:36,111 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9a7fdf8b82889b95543ffed36720442/A of d9a7fdf8b82889b95543ffed36720442 into 8ffb35ea50ba4a13a56cb2e035c8d628(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:36,111 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9a7fdf8b82889b95543ffed36720442: 2024-11-08T00:24:36,111 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., storeName=d9a7fdf8b82889b95543ffed36720442/A, priority=13, startTime=1731025476035; duration=0sec 2024-11-08T00:24:36,111 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:24:36,112 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9a7fdf8b82889b95543ffed36720442:A 2024-11-08T00:24:36,112 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:24:36,113 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:24:36,113 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): d9a7fdf8b82889b95543ffed36720442/C is initiating minor compaction (all files) 2024-11-08T00:24:36,113 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d9a7fdf8b82889b95543ffed36720442/C in TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:36,113 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/23fcb08a68f04acc890e87b6f5e74fc6, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/1d512628e2c14f19ab7a6ac5cdd7adde, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/8aac8454126f4a71b7b9a20d841e87ff] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp, totalSize=36.9 K 2024-11-08T00:24:36,114 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23fcb08a68f04acc890e87b6f5e74fc6, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731025470834 2024-11-08T00:24:36,114 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d512628e2c14f19ab7a6ac5cdd7adde, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731025472419 2024-11-08T00:24:36,115 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8aac8454126f4a71b7b9a20d841e87ff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1731025472593 2024-11-08T00:24:36,128 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d9a7fdf8b82889b95543ffed36720442#C#compaction#182 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:36,129 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/519b93a085a04f009ffa844b4c8b5bf7 is 50, key is test_row_0/C:col10/1731025472593/Put/seqid=0 2024-11-08T00:24:36,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742051_1227 (size=13255) 2024-11-08T00:24:36,467 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/2e7afa8fd5be4a558f0dedc2127cdf01 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/2e7afa8fd5be4a558f0dedc2127cdf01 2024-11-08T00:24:36,473 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9a7fdf8b82889b95543ffed36720442/B of d9a7fdf8b82889b95543ffed36720442 into 2e7afa8fd5be4a558f0dedc2127cdf01(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:36,473 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9a7fdf8b82889b95543ffed36720442: 2024-11-08T00:24:36,473 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., storeName=d9a7fdf8b82889b95543ffed36720442/B, priority=13, startTime=1731025476035; duration=0sec 2024-11-08T00:24:36,473 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:36,474 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9a7fdf8b82889b95543ffed36720442:B 2024-11-08T00:24:36,546 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/519b93a085a04f009ffa844b4c8b5bf7 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/519b93a085a04f009ffa844b4c8b5bf7 2024-11-08T00:24:36,550 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d9a7fdf8b82889b95543ffed36720442/C of d9a7fdf8b82889b95543ffed36720442 into 519b93a085a04f009ffa844b4c8b5bf7(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:36,550 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d9a7fdf8b82889b95543ffed36720442: 2024-11-08T00:24:36,550 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., storeName=d9a7fdf8b82889b95543ffed36720442/C, priority=13, startTime=1731025476035; duration=0sec 2024-11-08T00:24:36,551 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:36,551 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d9a7fdf8b82889b95543ffed36720442:C 2024-11-08T00:24:37,349 INFO [Thread-578 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-578. 2024-11-08T00:24:37,349 DEBUG [Thread-578 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T00:24:37,349 DEBUG [Thread-578 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:24:37,349 DEBUG [Thread-578 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:24:37,349 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:24:37,349 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-08T00:24:37,349 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-11-08T00:24:37,349 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 86 2024-11-08T00:24:37,349 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 29 2024-11-08T00:24:37,349 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 83 2024-11-08T00:24:37,349 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-11-08T00:24:37,349 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-08T00:24:37,349 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2622 2024-11-08T00:24:37,349 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2598 2024-11-08T00:24:37,349 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-08T00:24:37,349 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1179 2024-11-08T00:24:37,349 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3535 rows 2024-11-08T00:24:37,349 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1192 2024-11-08T00:24:37,349 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3576 rows 2024-11-08T00:24:37,350 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T00:24:37,350 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T00:24:37,350 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:24:37,350 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:24:37,350 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:24:37,352 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-08T00:24:37,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-08T00:24:37,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-08T00:24:37,355 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731025477355"}]},"ts":"1731025477355"} 2024-11-08T00:24:37,364 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-08T00:24:37,364 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-08T00:24:37,365 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-08T00:24:37,367 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d9a7fdf8b82889b95543ffed36720442, UNASSIGN}] 2024-11-08T00:24:37,369 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d9a7fdf8b82889b95543ffed36720442, UNASSIGN 2024-11-08T00:24:37,373 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=d9a7fdf8b82889b95543ffed36720442, regionState=CLOSING, regionLocation=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:37,375 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d9a7fdf8b82889b95543ffed36720442, UNASSIGN because future has completed 2024-11-08T00:24:37,375 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-08T00:24:37,375 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856}] 2024-11-08T00:24:37,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-08T00:24:37,528 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(122): Close d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:37,528 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-08T00:24:37,528 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1722): Closing d9a7fdf8b82889b95543ffed36720442, disabling compactions & flushes 2024-11-08T00:24:37,528 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:37,528 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:37,529 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. after waiting 1 ms 2024-11-08T00:24:37,529 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:37,529 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(2902): Flushing d9a7fdf8b82889b95543ffed36720442 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-08T00:24:37,536 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108f29591de0c3e46e0a834cdfa59c884c6_d9a7fdf8b82889b95543ffed36720442 is 50, key is test_row_0/A:col10/1731025477347/Put/seqid=0 2024-11-08T00:24:37,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742052_1228 (size=9914) 2024-11-08T00:24:37,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-08T00:24:37,941 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:37,945 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108f29591de0c3e46e0a834cdfa59c884c6_d9a7fdf8b82889b95543ffed36720442 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108f29591de0c3e46e0a834cdfa59c884c6_d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:37,947 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/98d2916f447d47c49639e5daa4ee2305, store: [table=TestAcidGuarantees family=A region=d9a7fdf8b82889b95543ffed36720442] 2024-11-08T00:24:37,947 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/98d2916f447d47c49639e5daa4ee2305 is 175, key is test_row_0/A:col10/1731025477347/Put/seqid=0 2024-11-08T00:24:37,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742053_1229 (size=22561) 2024-11-08T00:24:37,956 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=415, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/98d2916f447d47c49639e5daa4ee2305 2024-11-08T00:24:37,965 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/482f002fe5cc48f48cca7321ff6cc3a4 is 50, key is test_row_0/B:col10/1731025477347/Put/seqid=0 2024-11-08T00:24:37,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742054_1230 (size=9857) 2024-11-08T00:24:37,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-08T00:24:38,370 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/482f002fe5cc48f48cca7321ff6cc3a4 2024-11-08T00:24:38,379 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/58519d5aa24f4a319f408897cc2f6cb9 is 50, key is test_row_0/C:col10/1731025477347/Put/seqid=0 2024-11-08T00:24:38,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742055_1231 (size=9857) 2024-11-08T00:24:38,388 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/58519d5aa24f4a319f408897cc2f6cb9 2024-11-08T00:24:38,398 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/A/98d2916f447d47c49639e5daa4ee2305 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/98d2916f447d47c49639e5daa4ee2305 2024-11-08T00:24:38,413 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/98d2916f447d47c49639e5daa4ee2305, entries=100, sequenceid=415, filesize=22.0 K 2024-11-08T00:24:38,414 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/B/482f002fe5cc48f48cca7321ff6cc3a4 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/482f002fe5cc48f48cca7321ff6cc3a4 2024-11-08T00:24:38,420 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/482f002fe5cc48f48cca7321ff6cc3a4, entries=100, sequenceid=415, filesize=9.6 K 2024-11-08T00:24:38,422 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/.tmp/C/58519d5aa24f4a319f408897cc2f6cb9 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/58519d5aa24f4a319f408897cc2f6cb9 2024-11-08T00:24:38,428 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/58519d5aa24f4a319f408897cc2f6cb9, entries=100, sequenceid=415, filesize=9.6 K 2024-11-08T00:24:38,433 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for d9a7fdf8b82889b95543ffed36720442 in 903ms, sequenceid=415, compaction requested=false 2024-11-08T00:24:38,445 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/ce2b1e6d4cf84bd3803b3aa60b6b6a7b, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/2e23c1ee94804405839ddcab4fe4b70b, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/e00bd41cdad242deb1b41ac16e2e7938, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/d7019f8aba594c529714d6e2fc692aa6, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/62abf286e4c44f96bda5137b51942cf9, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/d7353164bca944c583fd746be5e6b983, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/f8bf53a234b441429bff84389f948ad7, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/35d1d98eaed844b684170d1f798f4a75, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/6520fcbbefda40ab8e67626c26b95942, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/c20f3863e3f54ed6b985d2d173c2071a, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/f2770cfa6d93487081c8d4307d34c30a, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/c3c92059e64b4824acf49903c9afa910, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/3f52825d19434d66bce1b29946082a16, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/2e6e6ce6df934eb4a51468985bcee6df, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/b72c455684734950a3d852166c965232, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/7cd57b92c73540b1b68b62114ae4eb40, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/330af41c337e4eafa720c451ed89da21, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/59f0605a3b36405795cb377ab8163cb9, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/165d0a3dbf764391af9708f44230c258, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/d2a954a055c646ca843115206ba785e4, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/b6f1eb805ef94f008b8be5b28c42206e, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/850ec7e709124c67b0dbdb6bb02a7a47, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/96734715b60e47149498d50c0b110a88, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/110645565ca74fe5bcb7bfb57e2ba990, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/9b1c33033f81419096bac1a7b035e0aa, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/1e4720efda4740df89b8f4ef5f0c334e, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/c1c8c0ad629e46bfa2cb7a284785f572, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/f363505d76b74ed2b0eda78e3d342a5a] to archive 2024-11-08T00:24:38,452 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T00:24:38,455 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/ce2b1e6d4cf84bd3803b3aa60b6b6a7b to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/ce2b1e6d4cf84bd3803b3aa60b6b6a7b 2024-11-08T00:24:38,457 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/2e23c1ee94804405839ddcab4fe4b70b to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/2e23c1ee94804405839ddcab4fe4b70b 2024-11-08T00:24:38,458 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/e00bd41cdad242deb1b41ac16e2e7938 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/e00bd41cdad242deb1b41ac16e2e7938 2024-11-08T00:24:38,460 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/d7019f8aba594c529714d6e2fc692aa6 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/d7019f8aba594c529714d6e2fc692aa6 2024-11-08T00:24:38,464 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/62abf286e4c44f96bda5137b51942cf9 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/62abf286e4c44f96bda5137b51942cf9 2024-11-08T00:24:38,466 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/d7353164bca944c583fd746be5e6b983 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/d7353164bca944c583fd746be5e6b983 2024-11-08T00:24:38,468 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/f8bf53a234b441429bff84389f948ad7 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/f8bf53a234b441429bff84389f948ad7 2024-11-08T00:24:38,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/35d1d98eaed844b684170d1f798f4a75 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/35d1d98eaed844b684170d1f798f4a75 2024-11-08T00:24:38,471 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/6520fcbbefda40ab8e67626c26b95942 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/6520fcbbefda40ab8e67626c26b95942 2024-11-08T00:24:38,473 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/c20f3863e3f54ed6b985d2d173c2071a to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/c20f3863e3f54ed6b985d2d173c2071a 2024-11-08T00:24:38,475 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/f2770cfa6d93487081c8d4307d34c30a to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/f2770cfa6d93487081c8d4307d34c30a 2024-11-08T00:24:38,478 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/c3c92059e64b4824acf49903c9afa910 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/c3c92059e64b4824acf49903c9afa910 2024-11-08T00:24:38,480 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/3f52825d19434d66bce1b29946082a16 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/3f52825d19434d66bce1b29946082a16 2024-11-08T00:24:38,482 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/2e6e6ce6df934eb4a51468985bcee6df to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/2e6e6ce6df934eb4a51468985bcee6df 2024-11-08T00:24:38,484 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/b72c455684734950a3d852166c965232 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/b72c455684734950a3d852166c965232 2024-11-08T00:24:38,485 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/7cd57b92c73540b1b68b62114ae4eb40 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/7cd57b92c73540b1b68b62114ae4eb40 2024-11-08T00:24:38,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-08T00:24:38,487 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/330af41c337e4eafa720c451ed89da21 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/330af41c337e4eafa720c451ed89da21 2024-11-08T00:24:38,488 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/59f0605a3b36405795cb377ab8163cb9 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/59f0605a3b36405795cb377ab8163cb9 2024-11-08T00:24:38,490 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/165d0a3dbf764391af9708f44230c258 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/165d0a3dbf764391af9708f44230c258 2024-11-08T00:24:38,492 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/d2a954a055c646ca843115206ba785e4 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/d2a954a055c646ca843115206ba785e4 2024-11-08T00:24:38,493 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/b6f1eb805ef94f008b8be5b28c42206e to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/b6f1eb805ef94f008b8be5b28c42206e 2024-11-08T00:24:38,495 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/850ec7e709124c67b0dbdb6bb02a7a47 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/850ec7e709124c67b0dbdb6bb02a7a47 2024-11-08T00:24:38,496 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/96734715b60e47149498d50c0b110a88 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/96734715b60e47149498d50c0b110a88 2024-11-08T00:24:38,497 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/110645565ca74fe5bcb7bfb57e2ba990 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/110645565ca74fe5bcb7bfb57e2ba990 2024-11-08T00:24:38,498 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/9b1c33033f81419096bac1a7b035e0aa to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/9b1c33033f81419096bac1a7b035e0aa 2024-11-08T00:24:38,500 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/1e4720efda4740df89b8f4ef5f0c334e to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/1e4720efda4740df89b8f4ef5f0c334e 2024-11-08T00:24:38,501 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/c1c8c0ad629e46bfa2cb7a284785f572 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/c1c8c0ad629e46bfa2cb7a284785f572 2024-11-08T00:24:38,502 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/f363505d76b74ed2b0eda78e3d342a5a to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/f363505d76b74ed2b0eda78e3d342a5a 2024-11-08T00:24:38,504 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/5089aa09a49243f98b336765b4ff6045, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/6a46e44a3c624f06ae4619b5306e28b1, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/d474b77f8af245ebb4a2c1f16b6102d1, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/ef6065e5dbce4632bf05235592f917c7, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/8d14730a23814202b8c6ca69e2a66cd5, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/2bba88877d7648dbac4de8fa5761b813, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/d943b4a8659e488c83c98344fb5d9901, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/2e0cc5a1dad64d3aafa01511ff3fc576, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/b498f3901f88434aa5d5e3e0827a75ab, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/0ef2538400844a77830924382dead922, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/7496209e0fb644b89f7a97478718c5cb, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/562fe23eef324d5291caf07717ee2536, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/d447a7ecf3b84904a20c866d1c2e8f9f, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/ad6999aa49f84559b4cf1520f8f057e7, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/c67eef1bb65b4e8b97569ed871a3d7a6, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/5363e0cf72954198b63b7665a2e65038, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/df498e57ef82475c9e2eff000adb55ca, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/d784fe6365b14a519d92d25e92c5ed67, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/0cf7fb5e11004dce9e3c6f4db1d2feb4, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/758e6a0394b0438180d4692f4eb8cf31, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/4f762fdb4472441f898edd06d687528b, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/a6573948946f43619a94c22f773fd1e6, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/66118748fa5b404f86e9b766e8d23571, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/a13744b89cae4b328dc31920c86b7494, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/d7cf8e26514b420385ace8b04adc3cc2, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/93c8483be8f04b228e0cdbac0bea73c5, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/ade81c1c31aa41d08070e0c8b65e6ab8, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/da0740e2e6b447a296856582bb484880] to archive 2024-11-08T00:24:38,505 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T00:24:38,507 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/5089aa09a49243f98b336765b4ff6045 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/5089aa09a49243f98b336765b4ff6045 2024-11-08T00:24:38,508 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/6a46e44a3c624f06ae4619b5306e28b1 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/6a46e44a3c624f06ae4619b5306e28b1 2024-11-08T00:24:38,510 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/d474b77f8af245ebb4a2c1f16b6102d1 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/d474b77f8af245ebb4a2c1f16b6102d1 2024-11-08T00:24:38,511 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/ef6065e5dbce4632bf05235592f917c7 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/ef6065e5dbce4632bf05235592f917c7 2024-11-08T00:24:38,512 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/8d14730a23814202b8c6ca69e2a66cd5 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/8d14730a23814202b8c6ca69e2a66cd5 2024-11-08T00:24:38,513 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/2bba88877d7648dbac4de8fa5761b813 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/2bba88877d7648dbac4de8fa5761b813 2024-11-08T00:24:38,515 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/d943b4a8659e488c83c98344fb5d9901 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/d943b4a8659e488c83c98344fb5d9901 2024-11-08T00:24:38,516 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/2e0cc5a1dad64d3aafa01511ff3fc576 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/2e0cc5a1dad64d3aafa01511ff3fc576 2024-11-08T00:24:38,518 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/b498f3901f88434aa5d5e3e0827a75ab to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/b498f3901f88434aa5d5e3e0827a75ab 2024-11-08T00:24:38,520 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/0ef2538400844a77830924382dead922 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/0ef2538400844a77830924382dead922 2024-11-08T00:24:38,521 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/7496209e0fb644b89f7a97478718c5cb to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/7496209e0fb644b89f7a97478718c5cb 2024-11-08T00:24:38,524 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/562fe23eef324d5291caf07717ee2536 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/562fe23eef324d5291caf07717ee2536 2024-11-08T00:24:38,526 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/d447a7ecf3b84904a20c866d1c2e8f9f to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/d447a7ecf3b84904a20c866d1c2e8f9f 2024-11-08T00:24:38,528 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/ad6999aa49f84559b4cf1520f8f057e7 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/ad6999aa49f84559b4cf1520f8f057e7 2024-11-08T00:24:38,529 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/c67eef1bb65b4e8b97569ed871a3d7a6 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/c67eef1bb65b4e8b97569ed871a3d7a6 2024-11-08T00:24:38,531 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/5363e0cf72954198b63b7665a2e65038 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/5363e0cf72954198b63b7665a2e65038 2024-11-08T00:24:38,533 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/df498e57ef82475c9e2eff000adb55ca to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/df498e57ef82475c9e2eff000adb55ca 2024-11-08T00:24:38,537 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/d784fe6365b14a519d92d25e92c5ed67 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/d784fe6365b14a519d92d25e92c5ed67 2024-11-08T00:24:38,539 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/0cf7fb5e11004dce9e3c6f4db1d2feb4 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/0cf7fb5e11004dce9e3c6f4db1d2feb4 2024-11-08T00:24:38,541 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/758e6a0394b0438180d4692f4eb8cf31 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/758e6a0394b0438180d4692f4eb8cf31 2024-11-08T00:24:38,542 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/4f762fdb4472441f898edd06d687528b to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/4f762fdb4472441f898edd06d687528b 2024-11-08T00:24:38,544 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/a6573948946f43619a94c22f773fd1e6 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/a6573948946f43619a94c22f773fd1e6 2024-11-08T00:24:38,546 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/66118748fa5b404f86e9b766e8d23571 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/66118748fa5b404f86e9b766e8d23571 2024-11-08T00:24:38,547 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/a13744b89cae4b328dc31920c86b7494 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/a13744b89cae4b328dc31920c86b7494 2024-11-08T00:24:38,548 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/d7cf8e26514b420385ace8b04adc3cc2 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/d7cf8e26514b420385ace8b04adc3cc2 2024-11-08T00:24:38,550 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/93c8483be8f04b228e0cdbac0bea73c5 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/93c8483be8f04b228e0cdbac0bea73c5 2024-11-08T00:24:38,551 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/ade81c1c31aa41d08070e0c8b65e6ab8 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/ade81c1c31aa41d08070e0c8b65e6ab8 2024-11-08T00:24:38,553 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/da0740e2e6b447a296856582bb484880 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/da0740e2e6b447a296856582bb484880 2024-11-08T00:24:38,554 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/644f453fa6b14d8baf64468e5ca7c77e, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/22d123ca509a48b4ae2398eda8759ae3, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/c20254017fcc4c4f8fe32e5e7732cddb, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/9860599e3f1c49038216cbca85e7060f, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/e056401ee9404ba285a3ed1f5498a85d, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/eb72ee21e11241ca8e8336d9318697a0, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/3f758ebb38a64836b8be66f3242334db, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/5a23e86eb26140179af08c7dc4e5e6b3, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/b9bd0fc1273547ba995445be0f248cb1, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/0e12d735e53a4afb9b6f6fbbc7c037fe, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/18d4731da1724941a326c96136344118, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/8e89857f066b49949897234ef98b4867, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/4a1e6ced5135485c842c43abf273984c, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/9a860f4c625c471c9f5ae0bef8aef4f4, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/a66a533b63a74516a34c4ad974467975, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/c57d5003ef924f06839ddc3d56dc18bf, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/27c19b18d7ff480ba0736c5451e95a1d, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/e94e1d87625648cca8cb8f2d7c2d59f4, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/b5fde4c2413c4f728d600b2577d705e6, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/c34732d20a754ed89640f8bba02d0f2a, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/2820577ac23c4d93aa7eea77c2e54a4e, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/8f83287868464ef189c3bd65a2e33257, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/cc18c26ec43e4c7b9073c54d4bf824c0, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/03b557cfc72248c5ae5844ced3c4df30, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/23fcb08a68f04acc890e87b6f5e74fc6, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/16b664ecf04b4978bcf04a8da60f3156, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/1d512628e2c14f19ab7a6ac5cdd7adde, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/8aac8454126f4a71b7b9a20d841e87ff] to archive 2024-11-08T00:24:38,557 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T00:24:38,559 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/644f453fa6b14d8baf64468e5ca7c77e to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/644f453fa6b14d8baf64468e5ca7c77e 2024-11-08T00:24:38,560 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/22d123ca509a48b4ae2398eda8759ae3 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/22d123ca509a48b4ae2398eda8759ae3 2024-11-08T00:24:38,562 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/c20254017fcc4c4f8fe32e5e7732cddb to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/c20254017fcc4c4f8fe32e5e7732cddb 2024-11-08T00:24:38,564 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/9860599e3f1c49038216cbca85e7060f to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/9860599e3f1c49038216cbca85e7060f 2024-11-08T00:24:38,566 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/e056401ee9404ba285a3ed1f5498a85d to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/e056401ee9404ba285a3ed1f5498a85d 2024-11-08T00:24:38,567 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/eb72ee21e11241ca8e8336d9318697a0 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/eb72ee21e11241ca8e8336d9318697a0 2024-11-08T00:24:38,569 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/3f758ebb38a64836b8be66f3242334db to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/3f758ebb38a64836b8be66f3242334db 2024-11-08T00:24:38,570 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/5a23e86eb26140179af08c7dc4e5e6b3 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/5a23e86eb26140179af08c7dc4e5e6b3 2024-11-08T00:24:38,572 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/b9bd0fc1273547ba995445be0f248cb1 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/b9bd0fc1273547ba995445be0f248cb1 2024-11-08T00:24:38,573 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/0e12d735e53a4afb9b6f6fbbc7c037fe to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/0e12d735e53a4afb9b6f6fbbc7c037fe 2024-11-08T00:24:38,575 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/18d4731da1724941a326c96136344118 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/18d4731da1724941a326c96136344118 2024-11-08T00:24:38,577 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/8e89857f066b49949897234ef98b4867 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/8e89857f066b49949897234ef98b4867 2024-11-08T00:24:38,578 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/4a1e6ced5135485c842c43abf273984c to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/4a1e6ced5135485c842c43abf273984c 2024-11-08T00:24:38,579 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/9a860f4c625c471c9f5ae0bef8aef4f4 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/9a860f4c625c471c9f5ae0bef8aef4f4 2024-11-08T00:24:38,581 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/a66a533b63a74516a34c4ad974467975 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/a66a533b63a74516a34c4ad974467975 2024-11-08T00:24:38,582 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/c57d5003ef924f06839ddc3d56dc18bf to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/c57d5003ef924f06839ddc3d56dc18bf 2024-11-08T00:24:38,583 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/27c19b18d7ff480ba0736c5451e95a1d to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/27c19b18d7ff480ba0736c5451e95a1d 2024-11-08T00:24:38,585 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/e94e1d87625648cca8cb8f2d7c2d59f4 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/e94e1d87625648cca8cb8f2d7c2d59f4 2024-11-08T00:24:38,587 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/b5fde4c2413c4f728d600b2577d705e6 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/b5fde4c2413c4f728d600b2577d705e6 2024-11-08T00:24:38,588 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/c34732d20a754ed89640f8bba02d0f2a to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/c34732d20a754ed89640f8bba02d0f2a 2024-11-08T00:24:38,598 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/2820577ac23c4d93aa7eea77c2e54a4e to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/2820577ac23c4d93aa7eea77c2e54a4e 2024-11-08T00:24:38,602 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/8f83287868464ef189c3bd65a2e33257 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/8f83287868464ef189c3bd65a2e33257 2024-11-08T00:24:38,604 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/cc18c26ec43e4c7b9073c54d4bf824c0 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/cc18c26ec43e4c7b9073c54d4bf824c0 2024-11-08T00:24:38,605 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/03b557cfc72248c5ae5844ced3c4df30 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/03b557cfc72248c5ae5844ced3c4df30 2024-11-08T00:24:38,607 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/23fcb08a68f04acc890e87b6f5e74fc6 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/23fcb08a68f04acc890e87b6f5e74fc6 2024-11-08T00:24:38,608 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/16b664ecf04b4978bcf04a8da60f3156 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/16b664ecf04b4978bcf04a8da60f3156 2024-11-08T00:24:38,609 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/1d512628e2c14f19ab7a6ac5cdd7adde to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/1d512628e2c14f19ab7a6ac5cdd7adde 2024-11-08T00:24:38,611 DEBUG [StoreCloser-TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/8aac8454126f4a71b7b9a20d841e87ff to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/8aac8454126f4a71b7b9a20d841e87ff 2024-11-08T00:24:38,616 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/recovered.edits/418.seqid, newMaxSeqId=418, maxSeqId=4 2024-11-08T00:24:38,618 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. 2024-11-08T00:24:38,618 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1676): Region close journal for d9a7fdf8b82889b95543ffed36720442: Waiting for close lock at 1731025477528Running coprocessor pre-close hooks at 1731025477528Disabling compacts and flushes for region at 1731025477528Disabling writes for close at 1731025477529 (+1 ms)Obtaining lock to block concurrent updates at 1731025477529Preparing flush snapshotting stores in d9a7fdf8b82889b95543ffed36720442 at 1731025477529Finished memstore snapshotting TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1731025477529Flushing stores of TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442. at 1731025477529Flushing d9a7fdf8b82889b95543ffed36720442/A: creating writer at 1731025477530 (+1 ms)Flushing mob file d9a7fdf8b82889b95543ffed36720442/A: appending metadata at 1731025477536 (+6 ms)Flushing mob file d9a7fdf8b82889b95543ffed36720442/A: closing flushed file at 1731025477536Flushing d9a7fdf8b82889b95543ffed36720442/A: appending metadata at 1731025477947 (+411 ms)Flushing d9a7fdf8b82889b95543ffed36720442/A: closing flushed file at 1731025477947Flushing d9a7fdf8b82889b95543ffed36720442/B: creating writer at 1731025477962 (+15 ms)Flushing d9a7fdf8b82889b95543ffed36720442/B: appending metadata at 1731025477965 (+3 ms)Flushing d9a7fdf8b82889b95543ffed36720442/B: closing flushed file at 1731025477965Flushing d9a7fdf8b82889b95543ffed36720442/C: creating writer at 1731025478376 (+411 ms)Flushing d9a7fdf8b82889b95543ffed36720442/C: appending metadata at 1731025478379 (+3 ms)Flushing d9a7fdf8b82889b95543ffed36720442/C: closing flushed file at 1731025478379Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@aaab35b: reopening flushed file at 1731025478395 (+16 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e1a113b: reopening flushed file at 1731025478413 (+18 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@50da0c47: reopening flushed file at 1731025478421 (+8 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for d9a7fdf8b82889b95543ffed36720442 in 903ms, sequenceid=415, compaction requested=false at 1731025478433 (+12 ms)Writing region close event to WAL at 1731025478613 (+180 ms)Running coprocessor post-close hooks at 1731025478617 (+4 ms)Closed at 1731025478618 (+1 ms) 2024-11-08T00:24:38,620 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(157): Closed d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:38,621 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=d9a7fdf8b82889b95543ffed36720442, regionState=CLOSED 2024-11-08T00:24:38,622 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 because future has completed 2024-11-08T00:24:38,625 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-08T00:24:38,625 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseRegionProcedure d9a7fdf8b82889b95543ffed36720442, server=3302f0f507bd,40479,1731025415856 in 1.2480 sec 2024-11-08T00:24:38,628 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=55 2024-11-08T00:24:38,629 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=55, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d9a7fdf8b82889b95543ffed36720442, UNASSIGN in 1.2590 sec 2024-11-08T00:24:38,632 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-08T00:24:38,632 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.2650 sec 2024-11-08T00:24:38,633 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731025478633"}]},"ts":"1731025478633"} 2024-11-08T00:24:38,635 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-08T00:24:38,635 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-08T00:24:38,637 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.2830 sec 2024-11-08T00:24:39,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-08T00:24:39,498 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:24:39,499 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-08T00:24:39,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T00:24:39,501 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T00:24:39,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-08T00:24:39,503 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=58, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T00:24:39,512 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:39,516 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A, FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B, FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C, FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/recovered.edits] 2024-11-08T00:24:39,519 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/8ffb35ea50ba4a13a56cb2e035c8d628 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/8ffb35ea50ba4a13a56cb2e035c8d628 2024-11-08T00:24:39,521 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/98d2916f447d47c49639e5daa4ee2305 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/A/98d2916f447d47c49639e5daa4ee2305 2024-11-08T00:24:39,524 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/2e7afa8fd5be4a558f0dedc2127cdf01 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/2e7afa8fd5be4a558f0dedc2127cdf01 2024-11-08T00:24:39,526 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/482f002fe5cc48f48cca7321ff6cc3a4 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/B/482f002fe5cc48f48cca7321ff6cc3a4 2024-11-08T00:24:39,528 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/519b93a085a04f009ffa844b4c8b5bf7 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/519b93a085a04f009ffa844b4c8b5bf7 2024-11-08T00:24:39,534 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/58519d5aa24f4a319f408897cc2f6cb9 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/C/58519d5aa24f4a319f408897cc2f6cb9 2024-11-08T00:24:39,538 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/recovered.edits/418.seqid to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442/recovered.edits/418.seqid 2024-11-08T00:24:39,539 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:39,540 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-08T00:24:39,540 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-08T00:24:39,541 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-08T00:24:39,547 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411082e94e9ee0a3841109bdccaac621cec2b_d9a7fdf8b82889b95543ffed36720442 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411082e94e9ee0a3841109bdccaac621cec2b_d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:39,548 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411083b3e57e15bd444068326a6e394f8cabb_d9a7fdf8b82889b95543ffed36720442 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411083b3e57e15bd444068326a6e394f8cabb_d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:39,550 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110865035815226d495584bbf90b7a9716b1_d9a7fdf8b82889b95543ffed36720442 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110865035815226d495584bbf90b7a9716b1_d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:39,553 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110869009e6a7d364b658ce0a50ede3511df_d9a7fdf8b82889b95543ffed36720442 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110869009e6a7d364b658ce0a50ede3511df_d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:39,555 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411087097ebcb69dc4dcfbdb699e84135cdd9_d9a7fdf8b82889b95543ffed36720442 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411087097ebcb69dc4dcfbdb699e84135cdd9_d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:39,556 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110882d91698e5a645b3a1dddfea859c43f3_d9a7fdf8b82889b95543ffed36720442 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110882d91698e5a645b3a1dddfea859c43f3_d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:39,558 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411088beceee55a47499f88f5ab3da4c70735_d9a7fdf8b82889b95543ffed36720442 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411088beceee55a47499f88f5ab3da4c70735_d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:39,559 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108981f0662c4614233a1964c70cfa74e0d_d9a7fdf8b82889b95543ffed36720442 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108981f0662c4614233a1964c70cfa74e0d_d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:39,561 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108b776d1e602954f2a8a793e27dbc0f37b_d9a7fdf8b82889b95543ffed36720442 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108b776d1e602954f2a8a793e27dbc0f37b_d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:39,563 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108ba23c1f3297144ef9a3116b90d001b43_d9a7fdf8b82889b95543ffed36720442 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108ba23c1f3297144ef9a3116b90d001b43_d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:39,564 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108bf8f22b38b7f4130b0cdefa72899cb28_d9a7fdf8b82889b95543ffed36720442 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108bf8f22b38b7f4130b0cdefa72899cb28_d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:39,567 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108c02b7b5ec1004224bcdc2bbc1501ad71_d9a7fdf8b82889b95543ffed36720442 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108c02b7b5ec1004224bcdc2bbc1501ad71_d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:39,569 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108d5b7854c750a47e188c680ba6c83f188_d9a7fdf8b82889b95543ffed36720442 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108d5b7854c750a47e188c680ba6c83f188_d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:39,570 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108e2213f9287694afd9fa188390bfa4041_d9a7fdf8b82889b95543ffed36720442 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108e2213f9287694afd9fa188390bfa4041_d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:39,573 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108e388e45138c34a93b758222b262c2eb6_d9a7fdf8b82889b95543ffed36720442 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108e388e45138c34a93b758222b262c2eb6_d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:39,575 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108e45c5b6f800849daa849a284a5e4e412_d9a7fdf8b82889b95543ffed36720442 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108e45c5b6f800849daa849a284a5e4e412_d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:39,577 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108e8b210ec99ca4b2397755414ebd07bae_d9a7fdf8b82889b95543ffed36720442 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108e8b210ec99ca4b2397755414ebd07bae_d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:39,579 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108eec4bfd559c74690bc57a84fa3e5e675_d9a7fdf8b82889b95543ffed36720442 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108eec4bfd559c74690bc57a84fa3e5e675_d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:39,581 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108f29591de0c3e46e0a834cdfa59c884c6_d9a7fdf8b82889b95543ffed36720442 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108f29591de0c3e46e0a834cdfa59c884c6_d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:39,582 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108f836d42cecfe43179ae86f9de3ee40db_d9a7fdf8b82889b95543ffed36720442 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108f836d42cecfe43179ae86f9de3ee40db_d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:39,584 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108fca2de3627b847e7986357943e43b554_d9a7fdf8b82889b95543ffed36720442 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108fca2de3627b847e7986357943e43b554_d9a7fdf8b82889b95543ffed36720442 2024-11-08T00:24:39,584 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-08T00:24:39,588 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=58, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T00:24:39,591 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-08T00:24:39,593 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-08T00:24:39,595 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=58, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T00:24:39,595 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-08T00:24:39,596 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731025479595"}]},"ts":"9223372036854775807"} 2024-11-08T00:24:39,599 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-08T00:24:39,599 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => d9a7fdf8b82889b95543ffed36720442, NAME => 'TestAcidGuarantees,,1731025451005.d9a7fdf8b82889b95543ffed36720442.', STARTKEY => '', ENDKEY => ''}] 2024-11-08T00:24:39,599 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-08T00:24:39,599 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731025479599"}]},"ts":"9223372036854775807"} 2024-11-08T00:24:39,602 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-08T00:24:39,603 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=58, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T00:24:39,604 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 104 msec 2024-11-08T00:24:39,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-08T00:24:39,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-08T00:24:39,609 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:24:39,620 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=242 (was 237) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1018944586_22 at /127.0.0.1:60600 [Waiting for operation #828] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1477324790_22 at /127.0.0.1:60586 [Waiting for operation #855] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c8e766d-da5e-64ca-5ccd-0c91fcfb6de3/cluster_72fb0791-5876-a833-2fa2-61645688b81f/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1018944586_22 at /127.0.0.1:45476 [Waiting for operation #993] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1477324790_22 at /127.0.0.1:38740 [Waiting for operation #1090] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c8e766d-da5e-64ca-5ccd-0c91fcfb6de3/cluster_72fb0791-5876-a833-2fa2-61645688b81f/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=467 (was 453) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=721 (was 682) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5081 (was 4469) - AvailableMemoryMB LEAK? - 2024-11-08T00:24:39,629 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=242, OpenFileDescriptor=467, MaxFileDescriptor=1048576, SystemLoadAverage=721, ProcessCount=11, AvailableMemoryMB=5079 2024-11-08T00:24:39,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-08T00:24:39,631 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T00:24:39,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-08T00:24:39,635 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T00:24:39,635 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:39,635 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 59 2024-11-08T00:24:39,637 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T00:24:39,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-08T00:24:39,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742056_1232 (size=959) 2024-11-08T00:24:39,659 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f 2024-11-08T00:24:39,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742057_1233 (size=53) 2024-11-08T00:24:39,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-08T00:24:39,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-08T00:24:40,079 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:24:40,079 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 94c140d3911f03ee9ce6997a067b1a83, disabling compactions & flushes 2024-11-08T00:24:40,079 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:40,079 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:40,079 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. after waiting 0 ms 2024-11-08T00:24:40,079 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:40,079 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:40,079 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 94c140d3911f03ee9ce6997a067b1a83: Waiting for close lock at 1731025480079Disabling compacts and flushes for region at 1731025480079Disabling writes for close at 1731025480079Writing region close event to WAL at 1731025480079Closed at 1731025480079 2024-11-08T00:24:40,081 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T00:24:40,081 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731025480081"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731025480081"}]},"ts":"1731025480081"} 2024-11-08T00:24:40,083 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-08T00:24:40,083 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T00:24:40,084 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731025480083"}]},"ts":"1731025480083"} 2024-11-08T00:24:40,085 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-08T00:24:40,086 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=94c140d3911f03ee9ce6997a067b1a83, ASSIGN}] 2024-11-08T00:24:40,087 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=94c140d3911f03ee9ce6997a067b1a83, ASSIGN 2024-11-08T00:24:40,088 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=94c140d3911f03ee9ce6997a067b1a83, ASSIGN; state=OFFLINE, location=3302f0f507bd,40479,1731025415856; forceNewPlan=false, retain=false 2024-11-08T00:24:40,238 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=94c140d3911f03ee9ce6997a067b1a83, regionState=OPENING, regionLocation=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:40,240 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=94c140d3911f03ee9ce6997a067b1a83, ASSIGN because future has completed 2024-11-08T00:24:40,241 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856}] 2024-11-08T00:24:40,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-08T00:24:40,396 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:40,396 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} 2024-11-08T00:24:40,397 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 94c140d3911f03ee9ce6997a067b1a83 2024-11-08T00:24:40,397 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:24:40,397 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7794): checking encryption for 94c140d3911f03ee9ce6997a067b1a83 2024-11-08T00:24:40,397 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7797): checking classloading for 94c140d3911f03ee9ce6997a067b1a83 2024-11-08T00:24:40,398 INFO [StoreOpener-94c140d3911f03ee9ce6997a067b1a83-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 94c140d3911f03ee9ce6997a067b1a83 2024-11-08T00:24:40,400 INFO [StoreOpener-94c140d3911f03ee9ce6997a067b1a83-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 94c140d3911f03ee9ce6997a067b1a83 columnFamilyName A 2024-11-08T00:24:40,400 DEBUG [StoreOpener-94c140d3911f03ee9ce6997a067b1a83-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:40,400 INFO [StoreOpener-94c140d3911f03ee9ce6997a067b1a83-1 {}] regionserver.HStore(327): Store=94c140d3911f03ee9ce6997a067b1a83/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:24:40,400 INFO [StoreOpener-94c140d3911f03ee9ce6997a067b1a83-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 94c140d3911f03ee9ce6997a067b1a83 2024-11-08T00:24:40,401 INFO [StoreOpener-94c140d3911f03ee9ce6997a067b1a83-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 94c140d3911f03ee9ce6997a067b1a83 columnFamilyName B 2024-11-08T00:24:40,401 DEBUG [StoreOpener-94c140d3911f03ee9ce6997a067b1a83-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:40,402 INFO [StoreOpener-94c140d3911f03ee9ce6997a067b1a83-1 {}] regionserver.HStore(327): Store=94c140d3911f03ee9ce6997a067b1a83/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:24:40,402 INFO [StoreOpener-94c140d3911f03ee9ce6997a067b1a83-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 94c140d3911f03ee9ce6997a067b1a83 2024-11-08T00:24:40,403 INFO [StoreOpener-94c140d3911f03ee9ce6997a067b1a83-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 94c140d3911f03ee9ce6997a067b1a83 columnFamilyName C 2024-11-08T00:24:40,403 DEBUG [StoreOpener-94c140d3911f03ee9ce6997a067b1a83-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:24:40,404 INFO [StoreOpener-94c140d3911f03ee9ce6997a067b1a83-1 {}] regionserver.HStore(327): Store=94c140d3911f03ee9ce6997a067b1a83/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:24:40,404 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1038): replaying wal for 94c140d3911f03ee9ce6997a067b1a83 2024-11-08T00:24:40,405 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83 2024-11-08T00:24:40,405 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83 2024-11-08T00:24:40,406 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1048): stopping wal replay for 94c140d3911f03ee9ce6997a067b1a83 2024-11-08T00:24:40,406 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for 94c140d3911f03ee9ce6997a067b1a83 2024-11-08T00:24:40,407 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-08T00:24:40,408 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1093): writing seq id for 94c140d3911f03ee9ce6997a067b1a83 2024-11-08T00:24:40,410 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T00:24:40,411 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1114): Opened 94c140d3911f03ee9ce6997a067b1a83; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71923023, jitterRate=0.07173655927181244}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-08T00:24:40,411 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 94c140d3911f03ee9ce6997a067b1a83 2024-11-08T00:24:40,412 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1006): Region open journal for 94c140d3911f03ee9ce6997a067b1a83: Running coprocessor pre-open hook at 1731025480397Writing region info on filesystem at 1731025480397Initializing all the Stores at 1731025480398 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731025480398Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731025480398Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731025480398Cleaning up temporary data from old regions at 1731025480406 (+8 ms)Running coprocessor post-open hooks at 1731025480411 (+5 ms)Region opened successfully at 1731025480412 (+1 ms) 2024-11-08T00:24:40,413 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., pid=61, masterSystemTime=1731025480393 2024-11-08T00:24:40,415 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:40,415 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:40,415 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=94c140d3911f03ee9ce6997a067b1a83, regionState=OPEN, openSeqNum=2, regionLocation=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:40,417 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 because future has completed 2024-11-08T00:24:40,420 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-08T00:24:40,420 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 in 177 msec 2024-11-08T00:24:40,422 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-11-08T00:24:40,423 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=94c140d3911f03ee9ce6997a067b1a83, ASSIGN in 334 msec 2024-11-08T00:24:40,424 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T00:24:40,424 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731025480424"}]},"ts":"1731025480424"} 2024-11-08T00:24:40,426 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-08T00:24:40,427 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T00:24:40,429 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 796 msec 2024-11-08T00:24:40,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-08T00:24:40,778 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:24:40,779 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@578a75e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:24:40,779 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:24:40,779 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:24:40,781 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:24:40,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:24:40,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:24:40,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@426a78dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:24:40,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:24:40,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:24:40,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:24:40,784 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51632, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:24:40,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13d9cf9e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:24:40,785 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:24:40,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:24:40,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:24:40,788 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45652, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:24:40,789 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-08T00:24:40,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 3302f0f507bd,36785,1731025414356 2024-11-08T00:24:40,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@76f2f122 2024-11-08T00:24:40,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-08T00:24:40,792 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51646, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-08T00:24:40,797 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b76211b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:24:40,797 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:24:40,797 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:24:40,799 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:24:40,799 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:24:40,799 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:24:40,799 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53367cf6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:24:40,799 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:24:40,800 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:24:40,800 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:24:40,801 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51658, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:24:40,802 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@363b582a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:24:40,813 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11b1e8a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:24:40,813 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:24:40,813 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:24:40,815 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:24:40,815 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:24:40,815 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:24:40,815 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5883bfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:24:40,816 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:24:40,816 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:24:40,816 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:24:40,818 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51684, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:24:40,819 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b42226f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:24:40,824 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@509857a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:24:40,825 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:24:40,825 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:24:40,826 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:24:40,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:24:40,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:24:40,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@311543be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:24:40,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:24:40,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:24:40,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:24:40,829 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51700, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:24:40,830 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e006d2d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:24:40,832 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1263b212, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:24:40,832 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:24:40,833 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:24:40,834 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:24:40,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:24:40,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:24:40,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7982b00e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:24:40,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:24:40,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:24:40,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:24:40,836 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51722, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:24:40,837 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36d822ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:24:40,842 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43295e22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:24:40,842 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:24:40,843 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:24:40,844 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:24:40,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:24:40,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:24:40,845 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@799e6abf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:24:40,845 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:24:40,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:24:40,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:24:40,847 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51744, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:24:40,848 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cd31b0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:24:40,850 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c19450f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:24:40,850 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:24:40,850 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:24:40,851 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:24:40,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:24:40,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:24:40,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a062741, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:24:40,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:24:40,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:24:40,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:24:40,854 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51768, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:24:40,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cc40a07, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:24:40,856 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21598d4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:24:40,856 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:24:40,856 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:24:40,858 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:24:40,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:24:40,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:24:40,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1348aeb9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:24:40,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:24:40,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:24:40,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:24:40,860 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51792, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:24:40,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c9799c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:24:40,864 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d7bbf3b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:24:40,864 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:24:40,865 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:24:40,866 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:24:40,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:24:40,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:24:40,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ab69000, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:24:40,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:24:40,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:24:40,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:24:40,868 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51806, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:24:40,869 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31e53826, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:24:40,874 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77ab4eba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:24:40,874 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:24:40,874 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:24:40,876 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:24:40,877 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:24:40,877 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:24:40,877 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d315524, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:24:40,877 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:24:40,877 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:24:40,878 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:24:40,880 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51830, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:24:40,881 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@490f1b76, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:24:40,884 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11b71713, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:24:40,884 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:24:40,885 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:24:40,886 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:24:40,886 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:24:40,886 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:24:40,886 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e60f45e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:24:40,886 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:24:40,887 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:24:40,887 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:24:40,888 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51842, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:24:40,893 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a17a80a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:24:40,894 DEBUG [Thread-1097 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:24:40,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:24:40,895 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:24:40,896 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45668, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:24:40,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2] 2024-11-08T00:24:40,900 DEBUG [Thread-1110 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:24:40,904 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:24:40,904 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:24:40,904 DEBUG [Thread-1091 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:24:40,906 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:24:40,906 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:24:40,909 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45670, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:24:40,911 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45682, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:24:40,912 DEBUG [Thread-1106 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:24:40,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2] 2024-11-08T00:24:40,914 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2] 2024-11-08T00:24:40,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:24:40,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:24:40,929 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45684, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:24:40,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2] 2024-11-08T00:24:40,941 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:24:40,941 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94c140d3911f03ee9ce6997a067b1a83 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T00:24:40,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 94c140d3911f03ee9ce6997a067b1a83 2024-11-08T00:24:40,948 DEBUG [Thread-1102 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:24:40,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:24:40,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:24:40,954 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45694, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:24:40,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2] 2024-11-08T00:24:40,958 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-08T00:24:40,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-08T00:24:40,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-08T00:24:40,976 DEBUG [Thread-1093 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:24:40,976 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T00:24:40,978 DEBUG [Thread-1093 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:24:40,979 DEBUG [Thread-1093 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:24:40,981 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45704, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:24:40,982 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2] 2024-11-08T00:24:40,984 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T00:24:40,984 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T00:24:40,992 DEBUG [Thread-1108 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:24:40,994 DEBUG [Thread-1108 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:24:40,994 DEBUG [Thread-1108 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:24:40,995 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45720, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:24:40,998 DEBUG [Thread-1108 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2] 2024-11-08T00:24:40,999 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/2fe06076275f4ea3833fad859f37d61a is 50, key is test_row_0/A:col10/1731025480940/Put/seqid=0 2024-11-08T00:24:41,013 DEBUG [Thread-1095 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:24:41,014 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:24:41,014 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:24:41,016 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45728, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:24:41,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2] 2024-11-08T00:24:41,035 DEBUG [Thread-1099 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:24:41,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:24:41,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:24:41,039 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45734, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:24:41,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2] 2024-11-08T00:24:41,052 DEBUG [Thread-1104 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:24:41,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:24:41,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:24:41,054 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45746, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:24:41,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2] 2024-11-08T00:24:41,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-08T00:24:41,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:41,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45704 deadline: 1731025541070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:41,096 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:41,096 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:41,096 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:41,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:41,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025541074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:41,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:41,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:41,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:41,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742058_1234 (size=12001) 2024-11-08T00:24:41,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:41,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025541096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:41,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:41,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:41,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:41,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:41,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025541096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:41,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:41,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:41,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:41,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:41,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025541089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:41,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:41,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:41,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:41,141 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-08T00:24:41,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:41,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. as already flushing 2024-11-08T00:24:41,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:41,146 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:41,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:41,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:41,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:41,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45704 deadline: 1731025541207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:41,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:41,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:41,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:41,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:41,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025541209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:41,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:41,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:41,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:41,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:41,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025541217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:41,227 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:41,227 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:41,227 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:41,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:41,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025541228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:41,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:41,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:41,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:41,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:41,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025541234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:41,240 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:41,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:41,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:41,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-08T00:24:41,314 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-08T00:24:41,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:41,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. as already flushing 2024-11-08T00:24:41,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:41,315 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:41,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:41,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:41,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:41,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45704 deadline: 1731025541427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:41,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:41,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025541427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:41,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:41,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:41,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:41,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:41,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:41,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:41,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:41,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025541439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:41,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:41,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:41,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:41,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:41,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025541447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:41,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:41,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:41,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:41,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:41,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025541449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:41,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:41,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:41,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:41,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-08T00:24:41,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:41,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. as already flushing 2024-11-08T00:24:41,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:41,468 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:41,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:41,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:41,497 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/2fe06076275f4ea3833fad859f37d61a 2024-11-08T00:24:41,568 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/7eaf26e8ac774e5a9f1df20ed379921c is 50, key is test_row_0/B:col10/1731025480940/Put/seqid=0 2024-11-08T00:24:41,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-08T00:24:41,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742059_1235 (size=12001) 2024-11-08T00:24:41,603 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/7eaf26e8ac774e5a9f1df20ed379921c 2024-11-08T00:24:41,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-08T00:24:41,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:41,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. as already flushing 2024-11-08T00:24:41,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:41,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:41,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:41,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:41,715 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/a3dea4bfe9924640972616982fae704d is 50, key is test_row_0/C:col10/1731025480940/Put/seqid=0 2024-11-08T00:24:41,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:41,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45704 deadline: 1731025541741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:41,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:41,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:41,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:41,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:41,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025541742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:41,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:41,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:41,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:41,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:41,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025541747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:41,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:41,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:41,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:41,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:41,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025541761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:41,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:41,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025541761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:41,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:41,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:41,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:41,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:41,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:41,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:41,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742060_1236 (size=12001) 2024-11-08T00:24:41,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-08T00:24:41,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:41,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. as already flushing 2024-11-08T00:24:41,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:41,785 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:41,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:41,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:41,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-08T00:24:41,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:41,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. as already flushing 2024-11-08T00:24:41,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:41,940 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:41,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:41,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:42,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-08T00:24:42,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:42,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. as already flushing 2024-11-08T00:24:42,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:42,096 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:42,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:42,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:42,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-08T00:24:42,176 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/a3dea4bfe9924640972616982fae704d 2024-11-08T00:24:42,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/2fe06076275f4ea3833fad859f37d61a as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/2fe06076275f4ea3833fad859f37d61a 2024-11-08T00:24:42,228 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/2fe06076275f4ea3833fad859f37d61a, entries=150, sequenceid=12, filesize=11.7 K 2024-11-08T00:24:42,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/7eaf26e8ac774e5a9f1df20ed379921c as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/7eaf26e8ac774e5a9f1df20ed379921c 2024-11-08T00:24:42,242 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/7eaf26e8ac774e5a9f1df20ed379921c, entries=150, sequenceid=12, filesize=11.7 K 2024-11-08T00:24:42,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/a3dea4bfe9924640972616982fae704d as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/a3dea4bfe9924640972616982fae704d 2024-11-08T00:24:42,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-08T00:24:42,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:42,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. as already flushing 2024-11-08T00:24:42,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:42,249 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:42,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:42,250 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/a3dea4bfe9924640972616982fae704d, entries=150, sequenceid=12, filesize=11.7 K 2024-11-08T00:24:42,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:42,252 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 94c140d3911f03ee9ce6997a067b1a83 in 1311ms, sequenceid=12, compaction requested=false 2024-11-08T00:24:42,252 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-08T00:24:42,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:24:42,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 94c140d3911f03ee9ce6997a067b1a83 2024-11-08T00:24:42,261 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:24:42,261 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94c140d3911f03ee9ce6997a067b1a83 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-08T00:24:42,269 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/320dd2f2da8d498dad74c26cbd0fdaf9 is 50, key is test_row_0/A:col10/1731025481088/Put/seqid=0 2024-11-08T00:24:42,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:42,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025542279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:42,286 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:42,286 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:42,286 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:42,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:42,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45704 deadline: 1731025542281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:42,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:42,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:42,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:42,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:42,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025542283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:42,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:42,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:42,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:42,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:42,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025542284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:42,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:42,291 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:42,291 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:42,291 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:42,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025542286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:42,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:42,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:42,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:42,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742061_1237 (size=14341) 2024-11-08T00:24:42,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-08T00:24:42,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:42,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. as already flushing 2024-11-08T00:24:42,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:42,423 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:42,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:42,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:42,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:42,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025542397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:42,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:42,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025542398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:42,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:42,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:42,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45704 deadline: 1731025542398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:42,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025542398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:42,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:42,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:42,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:42,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:42,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:42,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:42,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:42,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025542398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:42,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:42,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:42,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:42,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:42,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:42,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:42,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:42,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:42,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:42,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-08T00:24:42,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:42,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. as already flushing 2024-11-08T00:24:42,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:42,578 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:42,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:42,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:42,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:42,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025542637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:42,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:42,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:42,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:42,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:42,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025542638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:42,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:42,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:42,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:42,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:42,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:42,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025542638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:42,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45704 deadline: 1731025542638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:42,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:42,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:42,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:42,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:42,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:42,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:42,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:42,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025542638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:42,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:42,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:42,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:42,714 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/320dd2f2da8d498dad74c26cbd0fdaf9 2024-11-08T00:24:42,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/ec65a3f81da148bd90fcce853677dde3 is 50, key is test_row_0/B:col10/1731025481088/Put/seqid=0 2024-11-08T00:24:42,732 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-08T00:24:42,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:42,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. as already flushing 2024-11-08T00:24:42,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:42,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:42,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:42,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:42,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742062_1238 (size=12001) 2024-11-08T00:24:42,761 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/ec65a3f81da148bd90fcce853677dde3 2024-11-08T00:24:42,790 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/0b828357afdb4084a5d5105ac0c4f9b5 is 50, key is test_row_0/C:col10/1731025481088/Put/seqid=0 2024-11-08T00:24:42,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742063_1239 (size=12001) 2024-11-08T00:24:42,797 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/0b828357afdb4084a5d5105ac0c4f9b5 2024-11-08T00:24:42,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/320dd2f2da8d498dad74c26cbd0fdaf9 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/320dd2f2da8d498dad74c26cbd0fdaf9 2024-11-08T00:24:42,807 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/320dd2f2da8d498dad74c26cbd0fdaf9, entries=200, sequenceid=39, filesize=14.0 K 2024-11-08T00:24:42,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/ec65a3f81da148bd90fcce853677dde3 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/ec65a3f81da148bd90fcce853677dde3 2024-11-08T00:24:42,816 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/ec65a3f81da148bd90fcce853677dde3, entries=150, sequenceid=39, filesize=11.7 K 2024-11-08T00:24:42,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/0b828357afdb4084a5d5105ac0c4f9b5 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/0b828357afdb4084a5d5105ac0c4f9b5 2024-11-08T00:24:42,829 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/0b828357afdb4084a5d5105ac0c4f9b5, entries=150, sequenceid=39, filesize=11.7 K 2024-11-08T00:24:42,830 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for 94c140d3911f03ee9ce6997a067b1a83 in 569ms, sequenceid=39, compaction requested=false 2024-11-08T00:24:42,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:24:42,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-08T00:24:42,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:42,885 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 94c140d3911f03ee9ce6997a067b1a83 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-08T00:24:42,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/c8463c6a7b604d5784946940fe0c34a3 is 50, key is test_row_0/A:col10/1731025482278/Put/seqid=0 2024-11-08T00:24:42,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742064_1240 (size=12001) 2024-11-08T00:24:42,911 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/c8463c6a7b604d5784946940fe0c34a3 2024-11-08T00:24:42,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/a8427baf431745148d5852097d517d01 is 50, key is test_row_0/B:col10/1731025482278/Put/seqid=0 2024-11-08T00:24:42,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 94c140d3911f03ee9ce6997a067b1a83 2024-11-08T00:24:42,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. as already flushing 2024-11-08T00:24:42,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742065_1241 (size=12001) 2024-11-08T00:24:42,954 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/a8427baf431745148d5852097d517d01 2024-11-08T00:24:42,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/e781e280643642aba38501895335e94f is 50, key is test_row_0/C:col10/1731025482278/Put/seqid=0 2024-11-08T00:24:42,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:42,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025542985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:42,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:42,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:42,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:42,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:42,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025542988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:42,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:42,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:42,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:42,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:42,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025542993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:43,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:43,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:43,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:43,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:43,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025542995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:43,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:43,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:43,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:43,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:43,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45704 deadline: 1731025542995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:43,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:43,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:43,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:43,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742066_1242 (size=12001) 2024-11-08T00:24:43,011 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/e781e280643642aba38501895335e94f 2024-11-08T00:24:43,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/c8463c6a7b604d5784946940fe0c34a3 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/c8463c6a7b604d5784946940fe0c34a3 2024-11-08T00:24:43,028 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/c8463c6a7b604d5784946940fe0c34a3, entries=150, sequenceid=49, filesize=11.7 K 2024-11-08T00:24:43,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/a8427baf431745148d5852097d517d01 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/a8427baf431745148d5852097d517d01 2024-11-08T00:24:43,085 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/a8427baf431745148d5852097d517d01, entries=150, sequenceid=49, filesize=11.7 K 2024-11-08T00:24:43,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/e781e280643642aba38501895335e94f as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/e781e280643642aba38501895335e94f 2024-11-08T00:24:43,092 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/e781e280643642aba38501895335e94f, entries=150, sequenceid=49, filesize=11.7 K 2024-11-08T00:24:43,094 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=161.02 KB/164880 for 94c140d3911f03ee9ce6997a067b1a83 in 209ms, sequenceid=49, compaction requested=true 2024-11-08T00:24:43,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:24:43,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:43,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-08T00:24:43,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-08T00:24:43,098 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-08T00:24:43,099 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1120 sec 2024-11-08T00:24:43,100 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 2.1410 sec 2024-11-08T00:24:43,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-08T00:24:43,108 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:24:43,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 94c140d3911f03ee9ce6997a067b1a83 2024-11-08T00:24:43,109 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:24:43,109 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94c140d3911f03ee9ce6997a067b1a83 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-08T00:24:43,112 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-08T00:24:43,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-08T00:24:43,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-08T00:24:43,118 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T00:24:43,118 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T00:24:43,119 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T00:24:43,122 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/af40171d81d648788480eb61db332c79 is 50, key is test_row_0/A:col10/1731025483102/Put/seqid=0 2024-11-08T00:24:43,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:43,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025543115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:43,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:43,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025543116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:43,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:43,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:43,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:43,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:43,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:43,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:43,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:43,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025543118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:43,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:43,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025543126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:43,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:43,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45704 deadline: 1731025543127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:43,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:43,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:43,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:43,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:43,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:43,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:43,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:43,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:43,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:43,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742067_1243 (size=14341) 2024-11-08T00:24:43,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-08T00:24:43,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:43,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025543238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:43,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:43,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:43,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:43,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:43,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025543240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:43,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:43,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025543240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:43,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:43,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:43,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:43,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:43,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:43,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:43,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:43,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025543247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:43,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:43,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45704 deadline: 1731025543247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:43,255 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:43,255 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:43,255 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:43,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:43,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:43,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:43,280 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-08T00:24:43,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:43,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. as already flushing 2024-11-08T00:24:43,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:43,280 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:43,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:43,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:43,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-08T00:24:43,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-08T00:24:43,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:43,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. as already flushing 2024-11-08T00:24:43,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:43,438 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:43,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:43,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:43,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:43,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025543447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:43,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:43,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:43,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:43,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:43,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025543461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:43,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:43,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:43,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:43,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:43,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025543464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:43,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:43,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45704 deadline: 1731025543467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:43,473 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:43,473 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:43,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:43,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:43,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025543468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:43,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:43,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:43,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:43,477 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:43,477 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:43,477 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:43,584 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/af40171d81d648788480eb61db332c79 2024-11-08T00:24:43,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-08T00:24:43,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:43,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. as already flushing 2024-11-08T00:24:43,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:43,596 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:43,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:43,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:43,603 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/b058f83457784463978b70d227c0de6c is 50, key is test_row_0/B:col10/1731025483102/Put/seqid=0 2024-11-08T00:24:43,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742068_1244 (size=12001) 2024-11-08T00:24:43,646 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/b058f83457784463978b70d227c0de6c 2024-11-08T00:24:43,670 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/11469dca7a0d48f5a84d981160702c01 is 50, key is test_row_0/C:col10/1731025483102/Put/seqid=0 2024-11-08T00:24:43,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742069_1245 (size=12001) 2024-11-08T00:24:43,713 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/11469dca7a0d48f5a84d981160702c01 2024-11-08T00:24:43,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/af40171d81d648788480eb61db332c79 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/af40171d81d648788480eb61db332c79 2024-11-08T00:24:43,724 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/af40171d81d648788480eb61db332c79, entries=200, sequenceid=77, filesize=14.0 K 2024-11-08T00:24:43,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/b058f83457784463978b70d227c0de6c as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/b058f83457784463978b70d227c0de6c 2024-11-08T00:24:43,732 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/b058f83457784463978b70d227c0de6c, entries=150, sequenceid=77, filesize=11.7 K 2024-11-08T00:24:43,733 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/11469dca7a0d48f5a84d981160702c01 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/11469dca7a0d48f5a84d981160702c01 2024-11-08T00:24:43,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-08T00:24:43,738 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/11469dca7a0d48f5a84d981160702c01, entries=150, sequenceid=77, filesize=11.7 K 2024-11-08T00:24:43,739 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 94c140d3911f03ee9ce6997a067b1a83 in 630ms, sequenceid=77, compaction requested=true 2024-11-08T00:24:43,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:24:43,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c140d3911f03ee9ce6997a067b1a83:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T00:24:43,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:43,740 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T00:24:43,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c140d3911f03ee9ce6997a067b1a83:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T00:24:43,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:43,740 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T00:24:43,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c140d3911f03ee9ce6997a067b1a83:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:24:43,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:24:43,741 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T00:24:43,741 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): 94c140d3911f03ee9ce6997a067b1a83/B is initiating minor compaction (all files) 2024-11-08T00:24:43,742 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c140d3911f03ee9ce6997a067b1a83/B in TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:43,742 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/7eaf26e8ac774e5a9f1df20ed379921c, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/ec65a3f81da148bd90fcce853677dde3, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/a8427baf431745148d5852097d517d01, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/b058f83457784463978b70d227c0de6c] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp, totalSize=46.9 K 2024-11-08T00:24:43,742 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52684 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T00:24:43,742 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): 94c140d3911f03ee9ce6997a067b1a83/A is initiating minor compaction (all files) 2024-11-08T00:24:43,742 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c140d3911f03ee9ce6997a067b1a83/A in TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:43,742 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/2fe06076275f4ea3833fad859f37d61a, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/320dd2f2da8d498dad74c26cbd0fdaf9, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/c8463c6a7b604d5784946940fe0c34a3, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/af40171d81d648788480eb61db332c79] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp, totalSize=51.4 K 2024-11-08T00:24:43,743 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 7eaf26e8ac774e5a9f1df20ed379921c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731025480929 2024-11-08T00:24:43,743 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2fe06076275f4ea3833fad859f37d61a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731025480929 2024-11-08T00:24:43,744 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting ec65a3f81da148bd90fcce853677dde3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731025481088 2024-11-08T00:24:43,744 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 320dd2f2da8d498dad74c26cbd0fdaf9, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731025481086 2024-11-08T00:24:43,745 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8463c6a7b604d5784946940fe0c34a3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731025482278 2024-11-08T00:24:43,745 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting a8427baf431745148d5852097d517d01, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731025482278 2024-11-08T00:24:43,745 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting af40171d81d648788480eb61db332c79, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731025482983 2024-11-08T00:24:43,745 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting b058f83457784463978b70d227c0de6c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731025482983 2024-11-08T00:24:43,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-08T00:24:43,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:43,750 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 94c140d3911f03ee9ce6997a067b1a83 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-08T00:24:43,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/098d51f45b174fc298852c998b3b7687 is 50, key is test_row_0/A:col10/1731025483115/Put/seqid=0 2024-11-08T00:24:43,776 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c140d3911f03ee9ce6997a067b1a83#B#compaction#199 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:43,776 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/680c0d37aff34c15a7110e6de002d5c0 is 50, key is test_row_0/B:col10/1731025483102/Put/seqid=0 2024-11-08T00:24:43,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. as already flushing 2024-11-08T00:24:43,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 94c140d3911f03ee9ce6997a067b1a83 2024-11-08T00:24:43,786 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c140d3911f03ee9ce6997a067b1a83#A#compaction#200 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:43,787 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/fb939850917e458fa0eef9525ff7b74a is 50, key is test_row_0/A:col10/1731025483102/Put/seqid=0 2024-11-08T00:24:43,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742070_1246 (size=9657) 2024-11-08T00:24:43,819 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/098d51f45b174fc298852c998b3b7687 2024-11-08T00:24:43,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742071_1247 (size=12139) 2024-11-08T00:24:43,841 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/680c0d37aff34c15a7110e6de002d5c0 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/680c0d37aff34c15a7110e6de002d5c0 2024-11-08T00:24:43,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:43,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025543825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:43,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:43,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:43,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:43,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:43,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742072_1248 (size=12139) 2024-11-08T00:24:43,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45704 deadline: 1731025543825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:43,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:43,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025543828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:43,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:43,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:43,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:43,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:43,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:43,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:43,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:43,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025543842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:43,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/bd40ca65994e437081f2529e4862f983 is 50, key is test_row_0/B:col10/1731025483115/Put/seqid=0 2024-11-08T00:24:43,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:43,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:43,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:43,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025543842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:43,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:43,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:43,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:43,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:43,854 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 94c140d3911f03ee9ce6997a067b1a83/B of 94c140d3911f03ee9ce6997a067b1a83 into 680c0d37aff34c15a7110e6de002d5c0(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:43,855 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:24:43,855 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., storeName=94c140d3911f03ee9ce6997a067b1a83/B, priority=12, startTime=1731025483740; duration=0sec 2024-11-08T00:24:43,855 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:24:43,855 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c140d3911f03ee9ce6997a067b1a83:B 2024-11-08T00:24:43,855 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T00:24:43,857 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/fb939850917e458fa0eef9525ff7b74a as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/fb939850917e458fa0eef9525ff7b74a 2024-11-08T00:24:43,863 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T00:24:43,863 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): 94c140d3911f03ee9ce6997a067b1a83/C is initiating minor compaction (all files) 2024-11-08T00:24:43,863 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c140d3911f03ee9ce6997a067b1a83/C in TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:43,863 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/a3dea4bfe9924640972616982fae704d, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/0b828357afdb4084a5d5105ac0c4f9b5, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/e781e280643642aba38501895335e94f, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/11469dca7a0d48f5a84d981160702c01] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp, totalSize=46.9 K 2024-11-08T00:24:43,863 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting a3dea4bfe9924640972616982fae704d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731025480929 2024-11-08T00:24:43,864 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b828357afdb4084a5d5105ac0c4f9b5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731025481088 2024-11-08T00:24:43,864 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting e781e280643642aba38501895335e94f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731025482278 2024-11-08T00:24:43,865 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 11469dca7a0d48f5a84d981160702c01, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731025482983 2024-11-08T00:24:43,869 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 94c140d3911f03ee9ce6997a067b1a83/A of 94c140d3911f03ee9ce6997a067b1a83 into fb939850917e458fa0eef9525ff7b74a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:43,869 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:24:43,869 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., storeName=94c140d3911f03ee9ce6997a067b1a83/A, priority=12, startTime=1731025483739; duration=0sec 2024-11-08T00:24:43,869 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:43,869 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c140d3911f03ee9ce6997a067b1a83:A 2024-11-08T00:24:43,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742073_1249 (size=9657) 2024-11-08T00:24:43,888 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/bd40ca65994e437081f2529e4862f983 2024-11-08T00:24:43,896 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c140d3911f03ee9ce6997a067b1a83#C#compaction#202 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:43,896 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/e682d05e77a24a7c88bbeaf9ee5614f3 is 50, key is test_row_0/C:col10/1731025483102/Put/seqid=0 2024-11-08T00:24:43,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/c2a6d3a502ad42978f4f5a5529af7c76 is 50, key is test_row_0/C:col10/1731025483115/Put/seqid=0 2024-11-08T00:24:43,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742074_1250 (size=12139) 2024-11-08T00:24:43,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742075_1251 (size=9657) 2024-11-08T00:24:43,931 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/e682d05e77a24a7c88bbeaf9ee5614f3 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/e682d05e77a24a7c88bbeaf9ee5614f3 2024-11-08T00:24:43,937 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/c2a6d3a502ad42978f4f5a5529af7c76 2024-11-08T00:24:43,940 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 94c140d3911f03ee9ce6997a067b1a83/C of 94c140d3911f03ee9ce6997a067b1a83 into e682d05e77a24a7c88bbeaf9ee5614f3(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:43,940 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:24:43,940 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., storeName=94c140d3911f03ee9ce6997a067b1a83/C, priority=12, startTime=1731025483740; duration=0sec 2024-11-08T00:24:43,941 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:43,941 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c140d3911f03ee9ce6997a067b1a83:C 2024-11-08T00:24:43,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/098d51f45b174fc298852c998b3b7687 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/098d51f45b174fc298852c998b3b7687 2024-11-08T00:24:43,948 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/098d51f45b174fc298852c998b3b7687, entries=100, sequenceid=85, filesize=9.4 K 2024-11-08T00:24:43,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/bd40ca65994e437081f2529e4862f983 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/bd40ca65994e437081f2529e4862f983 2024-11-08T00:24:43,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:43,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025543948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:43,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:43,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:43,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45704 deadline: 1731025543948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:43,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:43,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:43,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:43,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:43,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:43,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:43,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025543948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:43,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:43,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:43,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:43,962 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/bd40ca65994e437081f2529e4862f983, entries=100, sequenceid=85, filesize=9.4 K 2024-11-08T00:24:43,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/c2a6d3a502ad42978f4f5a5529af7c76 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/c2a6d3a502ad42978f4f5a5529af7c76 2024-11-08T00:24:43,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:43,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025543957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:43,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:43,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025543957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:43,970 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/c2a6d3a502ad42978f4f5a5529af7c76, entries=100, sequenceid=85, filesize=9.4 K 2024-11-08T00:24:43,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:43,972 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:43,972 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:43,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:43,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:43,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:43,973 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=167.72 KB/171750 for 94c140d3911f03ee9ce6997a067b1a83 in 223ms, sequenceid=85, compaction requested=false 2024-11-08T00:24:43,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:24:43,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:43,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-08T00:24:43,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-08T00:24:43,977 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-08T00:24:43,978 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 856 msec 2024-11-08T00:24:43,981 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 867 msec 2024-11-08T00:24:44,144 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T00:24:44,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 94c140d3911f03ee9ce6997a067b1a83 2024-11-08T00:24:44,178 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:24:44,178 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94c140d3911f03ee9ce6997a067b1a83 3/3 column families, dataSize=187.85 KB heapSize=492.94 KB 2024-11-08T00:24:44,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:44,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45704 deadline: 1731025544181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:44,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:44,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025544181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:44,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:44,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:44,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:44,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:44,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025544179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:44,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:44,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:44,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:44,189 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:44,189 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:44,189 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:44,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:44,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025544187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:44,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:44,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:44,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:44,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:44,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025544189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:44,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:44,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:44,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:44,195 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/d669030354e24066bb345b6d0a8e96f2 is 50, key is test_row_0/A:col10/1731025484174/Put/seqid=0 2024-11-08T00:24:44,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742076_1252 (size=12001) 2024-11-08T00:24:44,227 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/d669030354e24066bb345b6d0a8e96f2 2024-11-08T00:24:44,236 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/ec27786c9ec045bab417e4fb5a59a844 is 50, key is test_row_0/B:col10/1731025484174/Put/seqid=0 2024-11-08T00:24:44,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-08T00:24:44,247 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:24:44,248 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-08T00:24:44,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-08T00:24:44,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-08T00:24:44,251 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T00:24:44,252 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T00:24:44,252 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T00:24:44,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742077_1253 (size=12001) 2024-11-08T00:24:44,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:44,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:44,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025544297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:44,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:44,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:44,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:44,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:44,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025544298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:44,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:44,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025544298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:44,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:44,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:44,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:44,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:44,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025544298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:44,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:44,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:44,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:44,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:44,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:44,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:44,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45704 deadline: 1731025544298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:44,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:44,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:44,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:44,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-08T00:24:44,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-08T00:24:44,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:44,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. as already flushing 2024-11-08T00:24:44,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:44,406 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:44,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:44,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:44,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:44,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025544508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:44,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:44,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025544508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:44,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:44,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:44,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025544508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:44,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:44,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:44,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:44,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025544508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:44,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:44,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:44,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:44,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:44,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:44,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:44,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:44,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:44,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:44,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:44,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45704 deadline: 1731025544517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:44,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:44,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:44,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:44,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-08T00:24:44,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-08T00:24:44,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:44,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. as already flushing 2024-11-08T00:24:44,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:44,580 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:44,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:44,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:44,671 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/ec27786c9ec045bab417e4fb5a59a844 2024-11-08T00:24:44,681 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/47db12face5a44d0ba3853429827ef92 is 50, key is test_row_0/C:col10/1731025484174/Put/seqid=0 2024-11-08T00:24:44,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742078_1254 (size=12001) 2024-11-08T00:24:44,700 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/47db12face5a44d0ba3853429827ef92 2024-11-08T00:24:44,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/d669030354e24066bb345b6d0a8e96f2 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/d669030354e24066bb345b6d0a8e96f2 2024-11-08T00:24:44,719 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/d669030354e24066bb345b6d0a8e96f2, entries=150, sequenceid=119, filesize=11.7 K 2024-11-08T00:24:44,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/ec27786c9ec045bab417e4fb5a59a844 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/ec27786c9ec045bab417e4fb5a59a844 2024-11-08T00:24:44,725 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/ec27786c9ec045bab417e4fb5a59a844, entries=150, sequenceid=119, filesize=11.7 K 2024-11-08T00:24:44,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/47db12face5a44d0ba3853429827ef92 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/47db12face5a44d0ba3853429827ef92 2024-11-08T00:24:44,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-08T00:24:44,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:44,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. as already flushing 2024-11-08T00:24:44,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:44,734 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:44,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:44,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:44,737 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/47db12face5a44d0ba3853429827ef92, entries=150, sequenceid=119, filesize=11.7 K 2024-11-08T00:24:44,738 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~187.85 KB/192360, heapSize ~492.89 KB/504720, currentSize=13.42 KB/13740 for 94c140d3911f03ee9ce6997a067b1a83 in 560ms, sequenceid=119, compaction requested=true 2024-11-08T00:24:44,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:24:44,738 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:24:44,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c140d3911f03ee9ce6997a067b1a83:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T00:24:44,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:44,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c140d3911f03ee9ce6997a067b1a83:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T00:24:44,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:44,738 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:24:44,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c140d3911f03ee9ce6997a067b1a83:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:24:44,738 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:24:44,740 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33797 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:24:44,740 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): 94c140d3911f03ee9ce6997a067b1a83/A is initiating minor compaction (all files) 2024-11-08T00:24:44,740 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c140d3911f03ee9ce6997a067b1a83/A in TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:44,740 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/fb939850917e458fa0eef9525ff7b74a, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/098d51f45b174fc298852c998b3b7687, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/d669030354e24066bb345b6d0a8e96f2] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp, totalSize=33.0 K 2024-11-08T00:24:44,740 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33797 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:24:44,740 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): 94c140d3911f03ee9ce6997a067b1a83/B is initiating minor compaction (all files) 2024-11-08T00:24:44,740 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c140d3911f03ee9ce6997a067b1a83/B in TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:44,740 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/680c0d37aff34c15a7110e6de002d5c0, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/bd40ca65994e437081f2529e4862f983, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/ec27786c9ec045bab417e4fb5a59a844] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp, totalSize=33.0 K 2024-11-08T00:24:44,741 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb939850917e458fa0eef9525ff7b74a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731025482983 2024-11-08T00:24:44,741 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 680c0d37aff34c15a7110e6de002d5c0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731025482983 2024-11-08T00:24:44,742 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 098d51f45b174fc298852c998b3b7687, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731025483115 2024-11-08T00:24:44,742 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting bd40ca65994e437081f2529e4862f983, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731025483115 2024-11-08T00:24:44,742 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting d669030354e24066bb345b6d0a8e96f2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731025483830 2024-11-08T00:24:44,742 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting ec27786c9ec045bab417e4fb5a59a844, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731025483830 2024-11-08T00:24:44,756 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c140d3911f03ee9ce6997a067b1a83#B#compaction#207 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:44,756 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/3c5f4cfdb6504851b4817986047095c7 is 50, key is test_row_0/B:col10/1731025484174/Put/seqid=0 2024-11-08T00:24:44,759 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c140d3911f03ee9ce6997a067b1a83#A#compaction#208 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:44,760 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/2d23523cc58342709ad8e73ef4657b6a is 50, key is test_row_0/A:col10/1731025484174/Put/seqid=0 2024-11-08T00:24:44,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742079_1255 (size=12241) 2024-11-08T00:24:44,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742080_1256 (size=12241) 2024-11-08T00:24:44,811 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/2d23523cc58342709ad8e73ef4657b6a as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/2d23523cc58342709ad8e73ef4657b6a 2024-11-08T00:24:44,819 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94c140d3911f03ee9ce6997a067b1a83/A of 94c140d3911f03ee9ce6997a067b1a83 into 2d23523cc58342709ad8e73ef4657b6a(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:44,819 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:24:44,819 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., storeName=94c140d3911f03ee9ce6997a067b1a83/A, priority=13, startTime=1731025484738; duration=0sec 2024-11-08T00:24:44,819 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:24:44,819 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c140d3911f03ee9ce6997a067b1a83:A 2024-11-08T00:24:44,819 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:24:44,821 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33797 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:24:44,821 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): 94c140d3911f03ee9ce6997a067b1a83/C is initiating minor compaction (all files) 2024-11-08T00:24:44,821 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c140d3911f03ee9ce6997a067b1a83/C in TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:44,821 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/e682d05e77a24a7c88bbeaf9ee5614f3, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/c2a6d3a502ad42978f4f5a5529af7c76, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/47db12face5a44d0ba3853429827ef92] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp, totalSize=33.0 K 2024-11-08T00:24:44,821 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting e682d05e77a24a7c88bbeaf9ee5614f3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731025482983 2024-11-08T00:24:44,822 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2a6d3a502ad42978f4f5a5529af7c76, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731025483115 2024-11-08T00:24:44,822 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47db12face5a44d0ba3853429827ef92, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731025483830 2024-11-08T00:24:44,832 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:24:44,832 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94c140d3911f03ee9ce6997a067b1a83 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-08T00:24:44,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 94c140d3911f03ee9ce6997a067b1a83 2024-11-08T00:24:44,836 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c140d3911f03ee9ce6997a067b1a83#C#compaction#209 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:44,836 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/291104c537dc44dcbc33d952254d7804 is 50, key is test_row_0/C:col10/1731025484174/Put/seqid=0 2024-11-08T00:24:44,840 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/3367ec7cf21b41c4a0d32d8fc8f2eb79 is 50, key is test_row_0/A:col10/1731025484824/Put/seqid=0 2024-11-08T00:24:44,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-08T00:24:44,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742081_1257 (size=12241) 2024-11-08T00:24:44,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742082_1258 (size=12151) 2024-11-08T00:24:44,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:44,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025544873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:44,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:44,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:44,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025544874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:44,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:44,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:44,885 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:44,885 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:44,885 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:44,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-08T00:24:44,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:44,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. as already flushing 2024-11-08T00:24:44,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:44,889 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:44,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:44,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:44,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:44,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025544884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:44,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:44,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45704 deadline: 1731025544884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:44,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:44,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:44,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:44,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:44,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:44,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:44,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:44,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025544884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:44,895 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:44,895 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:44,895 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:44,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:44,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025544988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:44,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:44,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:44,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:44,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025544988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:44,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:44,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:44,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:44,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:45,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:45,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:45,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025544997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:45,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45704 deadline: 1731025544997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:45,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:45,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:45,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:45,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025544997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:45,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:45,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:45,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:45,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:45,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:45,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:45,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:45,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-08T00:24:45,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:45,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. as already flushing 2024-11-08T00:24:45,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:45,042 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:45,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:45,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:45,194 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/3c5f4cfdb6504851b4817986047095c7 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/3c5f4cfdb6504851b4817986047095c7 2024-11-08T00:24:45,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-08T00:24:45,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:45,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. as already flushing 2024-11-08T00:24:45,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:45,196 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:45,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:45,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:45,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:45,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025545197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:45,203 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:45,203 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:45,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:45,203 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:45,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025545198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:45,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:45,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:45,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:45,204 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94c140d3911f03ee9ce6997a067b1a83/B of 94c140d3911f03ee9ce6997a067b1a83 into 3c5f4cfdb6504851b4817986047095c7(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:45,204 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:24:45,204 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., storeName=94c140d3911f03ee9ce6997a067b1a83/B, priority=13, startTime=1731025484738; duration=0sec 2024-11-08T00:24:45,204 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:45,204 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c140d3911f03ee9ce6997a067b1a83:B 2024-11-08T00:24:45,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:45,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45704 deadline: 1731025545207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:45,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:45,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:45,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:45,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:45,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025545207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:45,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:45,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:45,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:45,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:45,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025545207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:45,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:45,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:45,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:45,285 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/3367ec7cf21b41c4a0d32d8fc8f2eb79 2024-11-08T00:24:45,290 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/291104c537dc44dcbc33d952254d7804 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/291104c537dc44dcbc33d952254d7804 2024-11-08T00:24:45,302 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94c140d3911f03ee9ce6997a067b1a83/C of 94c140d3911f03ee9ce6997a067b1a83 into 291104c537dc44dcbc33d952254d7804(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:45,302 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:24:45,302 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., storeName=94c140d3911f03ee9ce6997a067b1a83/C, priority=13, startTime=1731025484738; duration=0sec 2024-11-08T00:24:45,302 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:45,302 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c140d3911f03ee9ce6997a067b1a83:C 2024-11-08T00:24:45,305 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/5731bf186df04e6aa4ab467282785238 is 50, key is test_row_0/B:col10/1731025484824/Put/seqid=0 2024-11-08T00:24:45,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742083_1259 (size=12151) 2024-11-08T00:24:45,326 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/5731bf186df04e6aa4ab467282785238 2024-11-08T00:24:45,346 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/bef16efff120483096c099e8c426d45a is 50, key is test_row_0/C:col10/1731025484824/Put/seqid=0 2024-11-08T00:24:45,350 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-08T00:24:45,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:45,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. as already flushing 2024-11-08T00:24:45,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:45,351 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:45,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:45,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:45,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742084_1260 (size=12151) 2024-11-08T00:24:45,381 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/bef16efff120483096c099e8c426d45a 2024-11-08T00:24:45,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-08T00:24:45,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/3367ec7cf21b41c4a0d32d8fc8f2eb79 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/3367ec7cf21b41c4a0d32d8fc8f2eb79 2024-11-08T00:24:45,397 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/3367ec7cf21b41c4a0d32d8fc8f2eb79, entries=150, sequenceid=132, filesize=11.9 K 2024-11-08T00:24:45,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/5731bf186df04e6aa4ab467282785238 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/5731bf186df04e6aa4ab467282785238 2024-11-08T00:24:45,425 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/5731bf186df04e6aa4ab467282785238, entries=150, sequenceid=132, filesize=11.9 K 2024-11-08T00:24:45,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/bef16efff120483096c099e8c426d45a as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/bef16efff120483096c099e8c426d45a 2024-11-08T00:24:45,431 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/bef16efff120483096c099e8c426d45a, entries=150, sequenceid=132, filesize=11.9 K 2024-11-08T00:24:45,432 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 94c140d3911f03ee9ce6997a067b1a83 in 600ms, sequenceid=132, compaction requested=false 2024-11-08T00:24:45,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:24:45,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-08T00:24:45,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:45,504 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 94c140d3911f03ee9ce6997a067b1a83 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-08T00:24:45,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/88294393938942c388fd8edf9ae8a9b3 is 50, key is test_row_0/A:col10/1731025484879/Put/seqid=0 2024-11-08T00:24:45,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 94c140d3911f03ee9ce6997a067b1a83 2024-11-08T00:24:45,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. as already flushing 2024-11-08T00:24:45,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742085_1261 (size=12151) 2024-11-08T00:24:45,547 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/88294393938942c388fd8edf9ae8a9b3 2024-11-08T00:24:45,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/9770c53929be45ce975bae97c8bbc39f is 50, key is test_row_0/B:col10/1731025484879/Put/seqid=0 2024-11-08T00:24:45,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:45,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025545546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:45,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:45,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45704 deadline: 1731025545546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:45,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:45,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:45,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:45,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:45,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:45,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:45,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025545546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:45,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:45,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:45,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:45,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:45,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:45,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025545553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:45,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:45,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:45,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:45,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:45,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025545557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:45,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:45,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:45,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:45,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742086_1262 (size=12151) 2024-11-08T00:24:45,571 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/9770c53929be45ce975bae97c8bbc39f 2024-11-08T00:24:45,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/4af8202f9ae6413cab00e4f155a6a67a is 50, key is test_row_0/C:col10/1731025484879/Put/seqid=0 2024-11-08T00:24:45,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742087_1263 (size=12151) 2024-11-08T00:24:45,586 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/4af8202f9ae6413cab00e4f155a6a67a 2024-11-08T00:24:45,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/88294393938942c388fd8edf9ae8a9b3 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/88294393938942c388fd8edf9ae8a9b3 2024-11-08T00:24:45,602 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/88294393938942c388fd8edf9ae8a9b3, entries=150, sequenceid=159, filesize=11.9 K 2024-11-08T00:24:45,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/9770c53929be45ce975bae97c8bbc39f as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/9770c53929be45ce975bae97c8bbc39f 2024-11-08T00:24:45,610 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/9770c53929be45ce975bae97c8bbc39f, entries=150, sequenceid=159, filesize=11.9 K 2024-11-08T00:24:45,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/4af8202f9ae6413cab00e4f155a6a67a as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/4af8202f9ae6413cab00e4f155a6a67a 2024-11-08T00:24:45,618 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/4af8202f9ae6413cab00e4f155a6a67a, entries=150, sequenceid=159, filesize=11.9 K 2024-11-08T00:24:45,622 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=67.09 KB/68700 for 94c140d3911f03ee9ce6997a067b1a83 in 118ms, sequenceid=159, compaction requested=true 2024-11-08T00:24:45,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:24:45,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:45,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-08T00:24:45,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-08T00:24:45,627 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-08T00:24:45,627 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3720 sec 2024-11-08T00:24:45,631 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.3800 sec 2024-11-08T00:24:45,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 94c140d3911f03ee9ce6997a067b1a83 2024-11-08T00:24:45,670 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:24:45,670 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94c140d3911f03ee9ce6997a067b1a83 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-08T00:24:45,675 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/1b0f19733e2b4e79bce06a36afab9eb7 is 50, key is test_row_0/A:col10/1731025485545/Put/seqid=0 2024-11-08T00:24:45,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742088_1264 (size=14541) 2024-11-08T00:24:45,683 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/1b0f19733e2b4e79bce06a36afab9eb7 2024-11-08T00:24:45,691 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/7bfe37849c6744a7bedc8880e69d4c5d is 50, key is test_row_0/B:col10/1731025485545/Put/seqid=0 2024-11-08T00:24:45,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742089_1265 (size=12151) 2024-11-08T00:24:45,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:45,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025545730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:45,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:45,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:45,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:45,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025545730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:45,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:45,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:45,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:45,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:45,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:45,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025545731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:45,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:45,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:45,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:45,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:45,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45704 deadline: 1731025545731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:45,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:45,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:45,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:45,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:45,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025545732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:45,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:45,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:45,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:45,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:45,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45704 deadline: 1731025545847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:45,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:45,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:45,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:45,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025545848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:45,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:45,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:45,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:45,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:45,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:45,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:45,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025545851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:45,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025545851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:45,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:45,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025545851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:45,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:45,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:45,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:45,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:45,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:45,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:45,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:45,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:45,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:46,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:46,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45704 deadline: 1731025546057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:46,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:46,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025546057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:46,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:46,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:46,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:46,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:46,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025546057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:46,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:46,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025546058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:46,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:46,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:46,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:46,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:46,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:46,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:46,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:46,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025546058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:46,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:46,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:46,065 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:46,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:46,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:46,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:46,095 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/7bfe37849c6744a7bedc8880e69d4c5d 2024-11-08T00:24:46,119 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/a154f2b5909f4e04bc0c31cf8f977942 is 50, key is test_row_0/C:col10/1731025485545/Put/seqid=0 2024-11-08T00:24:46,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742090_1266 (size=12151) 2024-11-08T00:24:46,178 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/a154f2b5909f4e04bc0c31cf8f977942 2024-11-08T00:24:46,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/1b0f19733e2b4e79bce06a36afab9eb7 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/1b0f19733e2b4e79bce06a36afab9eb7 2024-11-08T00:24:46,201 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/1b0f19733e2b4e79bce06a36afab9eb7, entries=200, sequenceid=173, filesize=14.2 K 2024-11-08T00:24:46,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/7bfe37849c6744a7bedc8880e69d4c5d as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/7bfe37849c6744a7bedc8880e69d4c5d 2024-11-08T00:24:46,213 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/7bfe37849c6744a7bedc8880e69d4c5d, entries=150, sequenceid=173, filesize=11.9 K 2024-11-08T00:24:46,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/a154f2b5909f4e04bc0c31cf8f977942 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/a154f2b5909f4e04bc0c31cf8f977942 2024-11-08T00:24:46,222 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/a154f2b5909f4e04bc0c31cf8f977942, entries=150, sequenceid=173, filesize=11.9 K 2024-11-08T00:24:46,223 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 94c140d3911f03ee9ce6997a067b1a83 in 553ms, sequenceid=173, compaction requested=true 2024-11-08T00:24:46,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:24:46,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c140d3911f03ee9ce6997a067b1a83:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T00:24:46,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:24:46,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c140d3911f03ee9ce6997a067b1a83:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T00:24:46,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-08T00:24:46,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c140d3911f03ee9ce6997a067b1a83:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:24:46,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-08T00:24:46,224 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T00:24:46,226 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51084 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T00:24:46,226 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): 94c140d3911f03ee9ce6997a067b1a83/A is initiating minor compaction (all files) 2024-11-08T00:24:46,226 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c140d3911f03ee9ce6997a067b1a83/A in TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:46,226 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/2d23523cc58342709ad8e73ef4657b6a, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/3367ec7cf21b41c4a0d32d8fc8f2eb79, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/88294393938942c388fd8edf9ae8a9b3, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/1b0f19733e2b4e79bce06a36afab9eb7] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp, totalSize=49.9 K 2024-11-08T00:24:46,227 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d23523cc58342709ad8e73ef4657b6a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731025483830 2024-11-08T00:24:46,227 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3367ec7cf21b41c4a0d32d8fc8f2eb79, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731025484824 2024-11-08T00:24:46,228 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88294393938942c388fd8edf9ae8a9b3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731025484869 2024-11-08T00:24:46,228 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T00:24:46,228 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b0f19733e2b4e79bce06a36afab9eb7, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731025485544 2024-11-08T00:24:46,244 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48694 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T00:24:46,244 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): 94c140d3911f03ee9ce6997a067b1a83/C is initiating minor compaction (all files) 2024-11-08T00:24:46,244 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c140d3911f03ee9ce6997a067b1a83/C in TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:46,245 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/291104c537dc44dcbc33d952254d7804, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/bef16efff120483096c099e8c426d45a, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/4af8202f9ae6413cab00e4f155a6a67a, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/a154f2b5909f4e04bc0c31cf8f977942] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp, totalSize=47.6 K 2024-11-08T00:24:46,246 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 291104c537dc44dcbc33d952254d7804, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731025483830 2024-11-08T00:24:46,251 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c140d3911f03ee9ce6997a067b1a83#A#compaction#219 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:46,252 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/ac62232696c9411ea2bf66e56573ae1c is 50, key is test_row_0/A:col10/1731025485545/Put/seqid=0 2024-11-08T00:24:46,252 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting bef16efff120483096c099e8c426d45a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731025484824 2024-11-08T00:24:46,256 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 4af8202f9ae6413cab00e4f155a6a67a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731025484869 2024-11-08T00:24:46,260 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting a154f2b5909f4e04bc0c31cf8f977942, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731025485545 2024-11-08T00:24:46,315 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c140d3911f03ee9ce6997a067b1a83#C#compaction#220 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:46,315 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/86dfa5eb208b460eb9a80641a7931538 is 50, key is test_row_0/C:col10/1731025485545/Put/seqid=0 2024-11-08T00:24:46,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742091_1267 (size=12527) 2024-11-08T00:24:46,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742092_1268 (size=12527) 2024-11-08T00:24:46,373 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/86dfa5eb208b460eb9a80641a7931538 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/86dfa5eb208b460eb9a80641a7931538 2024-11-08T00:24:46,376 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:24:46,376 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94c140d3911f03ee9ce6997a067b1a83 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-08T00:24:46,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 94c140d3911f03ee9ce6997a067b1a83 2024-11-08T00:24:46,382 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 94c140d3911f03ee9ce6997a067b1a83/C of 94c140d3911f03ee9ce6997a067b1a83 into 86dfa5eb208b460eb9a80641a7931538(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:46,382 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:24:46,382 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., storeName=94c140d3911f03ee9ce6997a067b1a83/C, priority=12, startTime=1731025486223; duration=0sec 2024-11-08T00:24:46,383 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:24:46,383 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c140d3911f03ee9ce6997a067b1a83:C 2024-11-08T00:24:46,383 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T00:24:46,384 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48694 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T00:24:46,385 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): 94c140d3911f03ee9ce6997a067b1a83/B is initiating minor compaction (all files) 2024-11-08T00:24:46,385 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c140d3911f03ee9ce6997a067b1a83/B in TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:46,385 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/3c5f4cfdb6504851b4817986047095c7, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/5731bf186df04e6aa4ab467282785238, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/9770c53929be45ce975bae97c8bbc39f, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/7bfe37849c6744a7bedc8880e69d4c5d] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp, totalSize=47.6 K 2024-11-08T00:24:46,385 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c5f4cfdb6504851b4817986047095c7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731025483830 2024-11-08T00:24:46,386 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 5731bf186df04e6aa4ab467282785238, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731025484824 2024-11-08T00:24:46,386 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 9770c53929be45ce975bae97c8bbc39f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731025484869 2024-11-08T00:24:46,386 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bfe37849c6744a7bedc8880e69d4c5d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731025485545 2024-11-08T00:24:46,387 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/dbd7b3c6b32b408c9417049cf57ecd8b is 50, key is test_row_0/A:col10/1731025485729/Put/seqid=0 2024-11-08T00:24:46,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-08T00:24:46,412 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:24:46,415 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c140d3911f03ee9ce6997a067b1a83#B#compaction#222 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:46,416 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/9f298e83c3cb42c98abae77d98309a07 is 50, key is test_row_0/B:col10/1731025485545/Put/seqid=0 2024-11-08T00:24:46,417 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-08T00:24:46,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-08T00:24:46,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-08T00:24:46,423 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T00:24:46,423 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T00:24:46,423 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T00:24:46,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:46,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:46,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025546407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:46,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025546407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:46,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:46,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:46,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:46,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:46,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:46,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:46,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:46,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025546436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:46,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:46,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45704 deadline: 1731025546437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:46,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:46,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:46,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:46,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:46,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025546437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:46,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:46,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:46,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:46,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:46,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:46,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:46,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742094_1270 (size=12527) 2024-11-08T00:24:46,462 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/9f298e83c3cb42c98abae77d98309a07 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/9f298e83c3cb42c98abae77d98309a07 2024-11-08T00:24:46,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742093_1269 (size=14541) 2024-11-08T00:24:46,471 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 94c140d3911f03ee9ce6997a067b1a83/B of 94c140d3911f03ee9ce6997a067b1a83 into 9f298e83c3cb42c98abae77d98309a07(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:46,471 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:24:46,471 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., storeName=94c140d3911f03ee9ce6997a067b1a83/B, priority=12, startTime=1731025486223; duration=0sec 2024-11-08T00:24:46,471 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:46,472 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c140d3911f03ee9ce6997a067b1a83:B 2024-11-08T00:24:46,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-08T00:24:46,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:46,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025546548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:46,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:46,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:46,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:46,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:46,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:46,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025546557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:46,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025546557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:46,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:46,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:46,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:46,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:46,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:46,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:46,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:46,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025546559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:46,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:46,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45704 deadline: 1731025546559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:46,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:46,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:46,564 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:46,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:46,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:46,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:46,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-08T00:24:46,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:46,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. as already flushing 2024-11-08T00:24:46,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:46,582 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:46,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:46,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:46,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-08T00:24:46,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-08T00:24:46,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:46,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. as already flushing 2024-11-08T00:24:46,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:46,740 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:46,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:46,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:46,746 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/ac62232696c9411ea2bf66e56573ae1c as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/ac62232696c9411ea2bf66e56573ae1c 2024-11-08T00:24:46,754 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 94c140d3911f03ee9ce6997a067b1a83/A of 94c140d3911f03ee9ce6997a067b1a83 into ac62232696c9411ea2bf66e56573ae1c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:46,754 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:24:46,754 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., storeName=94c140d3911f03ee9ce6997a067b1a83/A, priority=12, startTime=1731025486223; duration=0sec 2024-11-08T00:24:46,754 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:46,754 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c140d3911f03ee9ce6997a067b1a83:A 2024-11-08T00:24:46,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:46,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025546757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:46,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:46,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:46,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:46,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:46,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025546767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:46,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:46,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025546767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:46,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:46,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:46,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:46,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:46,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:46,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:46,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:46,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45704 deadline: 1731025546767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:46,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:46,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:46,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:46,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:46,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025546767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:46,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:46,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:46,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:46,871 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/dbd7b3c6b32b408c9417049cf57ecd8b 2024-11-08T00:24:46,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-08T00:24:46,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:46,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. as already flushing 2024-11-08T00:24:46,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:46,893 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:46,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:46,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:46,902 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/d41a21fb56104aaba98b82868179bfef is 50, key is test_row_0/B:col10/1731025485729/Put/seqid=0 2024-11-08T00:24:46,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742095_1271 (size=12151) 2024-11-08T00:24:46,938 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/d41a21fb56104aaba98b82868179bfef 2024-11-08T00:24:46,951 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/4bcd6a670e914dba814e7927de6d3852 is 50, key is test_row_0/C:col10/1731025485729/Put/seqid=0 2024-11-08T00:24:46,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742096_1272 (size=12151) 2024-11-08T00:24:46,977 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/4bcd6a670e914dba814e7927de6d3852 2024-11-08T00:24:46,988 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/dbd7b3c6b32b408c9417049cf57ecd8b as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/dbd7b3c6b32b408c9417049cf57ecd8b 2024-11-08T00:24:46,997 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/dbd7b3c6b32b408c9417049cf57ecd8b, entries=200, sequenceid=196, filesize=14.2 K 2024-11-08T00:24:46,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/d41a21fb56104aaba98b82868179bfef as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/d41a21fb56104aaba98b82868179bfef 2024-11-08T00:24:47,003 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/d41a21fb56104aaba98b82868179bfef, entries=150, sequenceid=196, filesize=11.9 K 2024-11-08T00:24:47,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/4bcd6a670e914dba814e7927de6d3852 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/4bcd6a670e914dba814e7927de6d3852 2024-11-08T00:24:47,014 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/4bcd6a670e914dba814e7927de6d3852, entries=150, sequenceid=196, filesize=11.9 K 2024-11-08T00:24:47,016 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 94c140d3911f03ee9ce6997a067b1a83 in 639ms, sequenceid=196, compaction requested=false 2024-11-08T00:24:47,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:24:47,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-08T00:24:47,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:47,047 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 94c140d3911f03ee9ce6997a067b1a83 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-08T00:24:47,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-08T00:24:47,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/25a375c163f244e8ad3f0146e3924841 is 50, key is test_row_0/A:col10/1731025486406/Put/seqid=0 2024-11-08T00:24:47,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 94c140d3911f03ee9ce6997a067b1a83 2024-11-08T00:24:47,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. as already flushing 2024-11-08T00:24:47,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742097_1273 (size=12151) 2024-11-08T00:24:47,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:47,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025547097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:47,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:47,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45704 deadline: 1731025547098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:47,103 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:47,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:47,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:47,103 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:47,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:47,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:47,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:47,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025547100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:47,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:47,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:47,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:47,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:47,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025547100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:47,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:47,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025547101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:47,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:47,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:47,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:47,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:47,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:47,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:47,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:47,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025547207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:47,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:47,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:47,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:47,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45704 deadline: 1731025547208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:47,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:47,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:47,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:47,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:47,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:47,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025547208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:47,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:47,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:47,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:47,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:47,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025547217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:47,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:47,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:47,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:47,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:47,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025547217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:47,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:47,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:47,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:47,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:47,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45704 deadline: 1731025547417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:47,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:47,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:47,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:47,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025547418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:47,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:47,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:47,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:47,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:47,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:47,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025547418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:47,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:47,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:47,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:47,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:47,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025547427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:47,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:47,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:47,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:47,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:47,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025547427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:47,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:47,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:47,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:47,480 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/25a375c163f244e8ad3f0146e3924841 2024-11-08T00:24:47,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/9715a1ad407648778dc152f0b7362dd7 is 50, key is test_row_0/B:col10/1731025486406/Put/seqid=0 2024-11-08T00:24:47,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742098_1274 (size=12151) 2024-11-08T00:24:47,506 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/9715a1ad407648778dc152f0b7362dd7 2024-11-08T00:24:47,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/53f48659d7aa4a4d9535a91beba306e9 is 50, key is test_row_0/C:col10/1731025486406/Put/seqid=0 2024-11-08T00:24:47,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742099_1275 (size=12151) 2024-11-08T00:24:47,523 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/53f48659d7aa4a4d9535a91beba306e9 2024-11-08T00:24:47,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/25a375c163f244e8ad3f0146e3924841 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/25a375c163f244e8ad3f0146e3924841 2024-11-08T00:24:47,533 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/25a375c163f244e8ad3f0146e3924841, entries=150, sequenceid=213, filesize=11.9 K 2024-11-08T00:24:47,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/9715a1ad407648778dc152f0b7362dd7 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/9715a1ad407648778dc152f0b7362dd7 2024-11-08T00:24:47,540 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/9715a1ad407648778dc152f0b7362dd7, entries=150, sequenceid=213, filesize=11.9 K 2024-11-08T00:24:47,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/53f48659d7aa4a4d9535a91beba306e9 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/53f48659d7aa4a4d9535a91beba306e9 2024-11-08T00:24:47,548 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/53f48659d7aa4a4d9535a91beba306e9, entries=150, sequenceid=213, filesize=11.9 K 2024-11-08T00:24:47,552 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 94c140d3911f03ee9ce6997a067b1a83 in 505ms, sequenceid=213, compaction requested=true 2024-11-08T00:24:47,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:24:47,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:47,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-08T00:24:47,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-08T00:24:47,556 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-08T00:24:47,556 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1310 sec 2024-11-08T00:24:47,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-08T00:24:47,560 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.1410 sec 2024-11-08T00:24:47,729 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:24:47,729 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94c140d3911f03ee9ce6997a067b1a83 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-08T00:24:47,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 94c140d3911f03ee9ce6997a067b1a83 2024-11-08T00:24:47,750 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/22c8b4063bee4cadbadc52d8b1908642 is 50, key is test_row_0/A:col10/1731025487729/Put/seqid=0 2024-11-08T00:24:47,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:47,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45704 deadline: 1731025547748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:47,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:47,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:47,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:47,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742100_1276 (size=14541) 2024-11-08T00:24:47,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:47,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025547750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:47,757 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/22c8b4063bee4cadbadc52d8b1908642 2024-11-08T00:24:47,757 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:47,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:47,757 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:47,757 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:47,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025547751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:47,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:47,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:47,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:47,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:47,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025547753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:47,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:47,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025547753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:47,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:47,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:47,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:47,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:47,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:47,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:47,768 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/e1d4fd9c50cd4918ad1924bb3627e80a is 50, key is test_row_0/B:col10/1731025487729/Put/seqid=0 2024-11-08T00:24:47,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742101_1277 (size=12151) 2024-11-08T00:24:47,773 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/e1d4fd9c50cd4918ad1924bb3627e80a 2024-11-08T00:24:47,809 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/48773eb5c0db402bb5e5828775fb2a61 is 50, key is test_row_0/C:col10/1731025487729/Put/seqid=0 2024-11-08T00:24:47,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742102_1278 (size=12151) 2024-11-08T00:24:47,836 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/48773eb5c0db402bb5e5828775fb2a61 2024-11-08T00:24:47,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/22c8b4063bee4cadbadc52d8b1908642 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/22c8b4063bee4cadbadc52d8b1908642 2024-11-08T00:24:47,849 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/22c8b4063bee4cadbadc52d8b1908642, entries=200, sequenceid=237, filesize=14.2 K 2024-11-08T00:24:47,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/e1d4fd9c50cd4918ad1924bb3627e80a as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/e1d4fd9c50cd4918ad1924bb3627e80a 2024-11-08T00:24:47,856 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/e1d4fd9c50cd4918ad1924bb3627e80a, entries=150, sequenceid=237, filesize=11.9 K 2024-11-08T00:24:47,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/48773eb5c0db402bb5e5828775fb2a61 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/48773eb5c0db402bb5e5828775fb2a61 2024-11-08T00:24:47,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:47,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45704 deadline: 1731025547858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:47,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:47,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:47,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:47,864 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/48773eb5c0db402bb5e5828775fb2a61, entries=150, sequenceid=237, filesize=11.9 K 2024-11-08T00:24:47,865 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 94c140d3911f03ee9ce6997a067b1a83 in 136ms, sequenceid=237, compaction requested=true 2024-11-08T00:24:47,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:24:47,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c140d3911f03ee9ce6997a067b1a83:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T00:24:47,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:47,865 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T00:24:47,865 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T00:24:47,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c140d3911f03ee9ce6997a067b1a83:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T00:24:47,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:47,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c140d3911f03ee9ce6997a067b1a83:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:24:47,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:24:47,866 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53760 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T00:24:47,867 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): 94c140d3911f03ee9ce6997a067b1a83/A is initiating minor compaction (all files) 2024-11-08T00:24:47,867 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T00:24:47,867 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): 94c140d3911f03ee9ce6997a067b1a83/B is initiating minor compaction (all files) 2024-11-08T00:24:47,867 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c140d3911f03ee9ce6997a067b1a83/A in TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:47,867 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c140d3911f03ee9ce6997a067b1a83/B in TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:47,867 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/ac62232696c9411ea2bf66e56573ae1c, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/dbd7b3c6b32b408c9417049cf57ecd8b, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/25a375c163f244e8ad3f0146e3924841, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/22c8b4063bee4cadbadc52d8b1908642] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp, totalSize=52.5 K 2024-11-08T00:24:47,867 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/9f298e83c3cb42c98abae77d98309a07, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/d41a21fb56104aaba98b82868179bfef, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/9715a1ad407648778dc152f0b7362dd7, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/e1d4fd9c50cd4918ad1924bb3627e80a] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp, totalSize=47.8 K 2024-11-08T00:24:47,868 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac62232696c9411ea2bf66e56573ae1c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731025485545 2024-11-08T00:24:47,869 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting dbd7b3c6b32b408c9417049cf57ecd8b, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731025485729 2024-11-08T00:24:47,869 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f298e83c3cb42c98abae77d98309a07, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731025485545 2024-11-08T00:24:47,869 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25a375c163f244e8ad3f0146e3924841, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731025486405 2024-11-08T00:24:47,869 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting d41a21fb56104aaba98b82868179bfef, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731025485729 2024-11-08T00:24:47,870 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22c8b4063bee4cadbadc52d8b1908642, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731025487095 2024-11-08T00:24:47,870 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 9715a1ad407648778dc152f0b7362dd7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731025486405 2024-11-08T00:24:47,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 94c140d3911f03ee9ce6997a067b1a83 2024-11-08T00:24:47,871 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:24:47,871 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94c140d3911f03ee9ce6997a067b1a83 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-08T00:24:47,871 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting e1d4fd9c50cd4918ad1924bb3627e80a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731025487099 2024-11-08T00:24:47,890 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/4bc09d38c8654924859dea99608a9c08 is 50, key is test_row_0/A:col10/1731025487869/Put/seqid=0 2024-11-08T00:24:47,903 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c140d3911f03ee9ce6997a067b1a83#B#compaction#233 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:47,903 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c140d3911f03ee9ce6997a067b1a83#A#compaction#232 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:47,903 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/8bb9bd8da7c045d7a5fb11cba35bd530 is 50, key is test_row_0/A:col10/1731025487729/Put/seqid=0 2024-11-08T00:24:47,903 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/46d937ee9f4e4b2f86fd7ed47d91d274 is 50, key is test_row_0/B:col10/1731025487729/Put/seqid=0 2024-11-08T00:24:47,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:47,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025547912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:47,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:47,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:47,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:47,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:47,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025547915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:47,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:47,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:47,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:47,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:47,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025547917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:47,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:47,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:47,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:47,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025547918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:47,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:47,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:47,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:47,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:47,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742103_1279 (size=14541) 2024-11-08T00:24:47,927 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/4bc09d38c8654924859dea99608a9c08 2024-11-08T00:24:47,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742104_1280 (size=12663) 2024-11-08T00:24:47,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/b44ad54bba97499bba82e7248f55fc74 is 50, key is test_row_0/B:col10/1731025487869/Put/seqid=0 2024-11-08T00:24:47,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742105_1281 (size=12663) 2024-11-08T00:24:47,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742106_1282 (size=12151) 2024-11-08T00:24:47,992 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/b44ad54bba97499bba82e7248f55fc74 2024-11-08T00:24:48,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/0c13273098c74711a8cb7b25311cfdc1 is 50, key is test_row_0/C:col10/1731025487869/Put/seqid=0 2024-11-08T00:24:48,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742107_1283 (size=12151) 2024-11-08T00:24:48,016 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/0c13273098c74711a8cb7b25311cfdc1 2024-11-08T00:24:48,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/4bc09d38c8654924859dea99608a9c08 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/4bc09d38c8654924859dea99608a9c08 2024-11-08T00:24:48,031 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/4bc09d38c8654924859dea99608a9c08, entries=200, sequenceid=250, filesize=14.2 K 2024-11-08T00:24:48,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:48,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:48,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025548028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:48,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025548028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:48,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:48,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:48,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:48,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:48,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:48,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:48,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:48,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:48,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025548028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:48,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025548028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:48,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:48,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:48,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:48,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:48,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:48,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:48,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/b44ad54bba97499bba82e7248f55fc74 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/b44ad54bba97499bba82e7248f55fc74 2024-11-08T00:24:48,039 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/b44ad54bba97499bba82e7248f55fc74, entries=150, sequenceid=250, filesize=11.9 K 2024-11-08T00:24:48,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/0c13273098c74711a8cb7b25311cfdc1 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/0c13273098c74711a8cb7b25311cfdc1 2024-11-08T00:24:48,047 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/0c13273098c74711a8cb7b25311cfdc1, entries=150, sequenceid=250, filesize=11.9 K 2024-11-08T00:24:48,049 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 94c140d3911f03ee9ce6997a067b1a83 in 178ms, sequenceid=250, compaction requested=true 2024-11-08T00:24:48,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:24:48,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c140d3911f03ee9ce6997a067b1a83:A, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:24:48,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-08T00:24:48,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c140d3911f03ee9ce6997a067b1a83:B, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:24:48,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-08T00:24:48,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c140d3911f03ee9ce6997a067b1a83:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:24:48,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-08T00:24:48,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 94c140d3911f03ee9ce6997a067b1a83 2024-11-08T00:24:48,072 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:24:48,072 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94c140d3911f03ee9ce6997a067b1a83 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-08T00:24:48,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/95128cd9c9684670a2f38ffe2269802d is 50, key is test_row_0/A:col10/1731025487915/Put/seqid=0 2024-11-08T00:24:48,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742108_1284 (size=14741) 2024-11-08T00:24:48,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:48,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45704 deadline: 1731025548143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:48,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:48,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:48,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:48,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:48,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025548237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:48,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:48,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:48,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:48,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025548237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:48,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:48,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:48,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:48,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:48,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:48,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025548237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:48,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:48,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:48,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:48,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:48,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025548237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:48,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:48,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:48,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:48,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:48,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45704 deadline: 1731025548257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:48,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:48,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:48,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:48,361 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/46d937ee9f4e4b2f86fd7ed47d91d274 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/46d937ee9f4e4b2f86fd7ed47d91d274 2024-11-08T00:24:48,377 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 94c140d3911f03ee9ce6997a067b1a83/B of 94c140d3911f03ee9ce6997a067b1a83 into 46d937ee9f4e4b2f86fd7ed47d91d274(size=12.4 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:48,377 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:24:48,377 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., storeName=94c140d3911f03ee9ce6997a067b1a83/B, priority=12, startTime=1731025487865; duration=0sec 2024-11-08T00:24:48,377 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-08T00:24:48,377 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c140d3911f03ee9ce6997a067b1a83:B 2024-11-08T00:24:48,377 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 4 compacting, 1 eligible, 16 blocking 2024-11-08T00:24:48,379 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/8bb9bd8da7c045d7a5fb11cba35bd530 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/8bb9bd8da7c045d7a5fb11cba35bd530 2024-11-08T00:24:48,383 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-08T00:24:48,383 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-08T00:24:48,384 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. because compaction request was cancelled 2024-11-08T00:24:48,384 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c140d3911f03ee9ce6997a067b1a83:A 2024-11-08T00:24:48,385 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-08T00:24:48,386 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 94c140d3911f03ee9ce6997a067b1a83/A of 94c140d3911f03ee9ce6997a067b1a83 into 8bb9bd8da7c045d7a5fb11cba35bd530(size=12.4 K), total size for store is 26.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:48,386 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:24:48,386 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., storeName=94c140d3911f03ee9ce6997a067b1a83/A, priority=12, startTime=1731025487865; duration=0sec 2024-11-08T00:24:48,386 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-08T00:24:48,386 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c140d3911f03ee9ce6997a067b1a83:A 2024-11-08T00:24:48,386 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c140d3911f03ee9ce6997a067b1a83:B 2024-11-08T00:24:48,387 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61131 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-08T00:24:48,388 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): 94c140d3911f03ee9ce6997a067b1a83/C is initiating minor compaction (all files) 2024-11-08T00:24:48,388 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c140d3911f03ee9ce6997a067b1a83/C in TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:48,388 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/86dfa5eb208b460eb9a80641a7931538, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/4bcd6a670e914dba814e7927de6d3852, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/53f48659d7aa4a4d9535a91beba306e9, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/48773eb5c0db402bb5e5828775fb2a61, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/0c13273098c74711a8cb7b25311cfdc1] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp, totalSize=59.7 K 2024-11-08T00:24:48,388 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 5 compacting, 0 eligible, 16 blocking 2024-11-08T00:24:48,388 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-08T00:24:48,388 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-08T00:24:48,388 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. because compaction request was cancelled 2024-11-08T00:24:48,388 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c140d3911f03ee9ce6997a067b1a83:C 2024-11-08T00:24:48,388 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-08T00:24:48,389 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-08T00:24:48,389 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-08T00:24:48,389 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. because compaction request was cancelled 2024-11-08T00:24:48,389 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c140d3911f03ee9ce6997a067b1a83:B 2024-11-08T00:24:48,389 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 86dfa5eb208b460eb9a80641a7931538, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731025485545 2024-11-08T00:24:48,390 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bcd6a670e914dba814e7927de6d3852, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731025485729 2024-11-08T00:24:48,391 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 53f48659d7aa4a4d9535a91beba306e9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731025486405 2024-11-08T00:24:48,392 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 48773eb5c0db402bb5e5828775fb2a61, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731025487099 2024-11-08T00:24:48,392 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c13273098c74711a8cb7b25311cfdc1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731025487743 2024-11-08T00:24:48,413 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c140d3911f03ee9ce6997a067b1a83#C#compaction#237 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:48,413 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/0af7a379dd804196ade4bf97a9ff1686 is 50, key is test_row_0/C:col10/1731025487869/Put/seqid=0 2024-11-08T00:24:48,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742109_1285 (size=12697) 2024-11-08T00:24:48,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:48,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45704 deadline: 1731025548467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:48,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:48,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:48,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:48,538 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/95128cd9c9684670a2f38ffe2269802d 2024-11-08T00:24:48,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/45b55ac89efe4221bf83a1747aada78d is 50, key is test_row_0/B:col10/1731025487915/Put/seqid=0 2024-11-08T00:24:48,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:48,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025548547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:48,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:48,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:48,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:48,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025548548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:48,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:48,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:48,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:48,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:48,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:48,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025548548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:48,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:48,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:48,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:48,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:48,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025548548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:48,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:48,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:48,552 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:48,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-08T00:24:48,567 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:24:48,567 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-08T00:24:48,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-08T00:24:48,570 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T00:24:48,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-08T00:24:48,571 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T00:24:48,571 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T00:24:48,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742110_1286 (size=12301) 2024-11-08T00:24:48,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-08T00:24:48,732 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-08T00:24:48,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:48,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. as already flushing 2024-11-08T00:24:48,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:48,733 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:48,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:48,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:48,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:48,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45704 deadline: 1731025548778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:48,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:48,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:48,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:48,844 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/0af7a379dd804196ade4bf97a9ff1686 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/0af7a379dd804196ade4bf97a9ff1686 2024-11-08T00:24:48,853 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 94c140d3911f03ee9ce6997a067b1a83/C of 94c140d3911f03ee9ce6997a067b1a83 into 0af7a379dd804196ade4bf97a9ff1686(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:48,853 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:24:48,853 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., storeName=94c140d3911f03ee9ce6997a067b1a83/C, priority=11, startTime=1731025488050; duration=0sec 2024-11-08T00:24:48,853 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:48,853 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c140d3911f03ee9ce6997a067b1a83:C 2024-11-08T00:24:48,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-08T00:24:48,919 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-08T00:24:48,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:48,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. as already flushing 2024-11-08T00:24:48,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:48,919 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:48,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:48,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:49,019 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/45b55ac89efe4221bf83a1747aada78d 2024-11-08T00:24:49,030 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/49fd83d260624c0585fe344a52bd09cf is 50, key is test_row_0/C:col10/1731025487915/Put/seqid=0 2024-11-08T00:24:49,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:49,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:49,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025549058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:49,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:49,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025549060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:49,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:49,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:49,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:49,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025549058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:49,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:49,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:49,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:49,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:49,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:49,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:49,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-08T00:24:49,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:49,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:49,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025549067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:49,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. as already flushing 2024-11-08T00:24:49,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:49,073 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:49,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:49,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742111_1287 (size=12301) 2024-11-08T00:24:49,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:49,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:49,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:49,074 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/49fd83d260624c0585fe344a52bd09cf 2024-11-08T00:24:49,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:49,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/95128cd9c9684670a2f38ffe2269802d as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/95128cd9c9684670a2f38ffe2269802d 2024-11-08T00:24:49,111 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/95128cd9c9684670a2f38ffe2269802d, entries=200, sequenceid=274, filesize=14.4 K 2024-11-08T00:24:49,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/45b55ac89efe4221bf83a1747aada78d as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/45b55ac89efe4221bf83a1747aada78d 2024-11-08T00:24:49,117 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/45b55ac89efe4221bf83a1747aada78d, entries=150, sequenceid=274, filesize=12.0 K 2024-11-08T00:24:49,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/49fd83d260624c0585fe344a52bd09cf as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/49fd83d260624c0585fe344a52bd09cf 2024-11-08T00:24:49,127 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/49fd83d260624c0585fe344a52bd09cf, entries=150, sequenceid=274, filesize=12.0 K 2024-11-08T00:24:49,136 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 94c140d3911f03ee9ce6997a067b1a83 in 1064ms, sequenceid=274, compaction requested=true 2024-11-08T00:24:49,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:24:49,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c140d3911f03ee9ce6997a067b1a83:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T00:24:49,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:49,137 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:24:49,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c140d3911f03ee9ce6997a067b1a83:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T00:24:49,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:24:49,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c140d3911f03ee9ce6997a067b1a83:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:24:49,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-08T00:24:49,137 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:24:49,138 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41945 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:24:49,138 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:24:49,138 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): 94c140d3911f03ee9ce6997a067b1a83/A is initiating minor compaction (all files) 2024-11-08T00:24:49,138 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): 94c140d3911f03ee9ce6997a067b1a83/B is initiating minor compaction (all files) 2024-11-08T00:24:49,138 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c140d3911f03ee9ce6997a067b1a83/A in TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:49,138 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c140d3911f03ee9ce6997a067b1a83/B in TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:49,138 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/8bb9bd8da7c045d7a5fb11cba35bd530, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/4bc09d38c8654924859dea99608a9c08, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/95128cd9c9684670a2f38ffe2269802d] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp, totalSize=41.0 K 2024-11-08T00:24:49,138 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/46d937ee9f4e4b2f86fd7ed47d91d274, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/b44ad54bba97499bba82e7248f55fc74, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/45b55ac89efe4221bf83a1747aada78d] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp, totalSize=36.2 K 2024-11-08T00:24:49,138 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bb9bd8da7c045d7a5fb11cba35bd530, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731025487099 2024-11-08T00:24:49,138 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 46d937ee9f4e4b2f86fd7ed47d91d274, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731025487099 2024-11-08T00:24:49,139 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4bc09d38c8654924859dea99608a9c08, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731025487743 2024-11-08T00:24:49,139 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting b44ad54bba97499bba82e7248f55fc74, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731025487743 2024-11-08T00:24:49,139 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95128cd9c9684670a2f38ffe2269802d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731025487907 2024-11-08T00:24:49,139 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 45b55ac89efe4221bf83a1747aada78d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731025487907 2024-11-08T00:24:49,158 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c140d3911f03ee9ce6997a067b1a83#B#compaction#240 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:49,159 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/2b7a9e9d2ca5447ea13eb39151206822 is 50, key is test_row_0/B:col10/1731025487915/Put/seqid=0 2024-11-08T00:24:49,178 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c140d3911f03ee9ce6997a067b1a83#A#compaction#241 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:49,179 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/2ab8a02378eb447f8bedd75dad2f87f3 is 50, key is test_row_0/A:col10/1731025487915/Put/seqid=0 2024-11-08T00:24:49,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-08T00:24:49,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742112_1288 (size=12915) 2024-11-08T00:24:49,224 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/2b7a9e9d2ca5447ea13eb39151206822 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/2b7a9e9d2ca5447ea13eb39151206822 2024-11-08T00:24:49,230 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94c140d3911f03ee9ce6997a067b1a83/B of 94c140d3911f03ee9ce6997a067b1a83 into 2b7a9e9d2ca5447ea13eb39151206822(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:49,230 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:24:49,230 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., storeName=94c140d3911f03ee9ce6997a067b1a83/B, priority=13, startTime=1731025489137; duration=0sec 2024-11-08T00:24:49,230 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:24:49,230 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c140d3911f03ee9ce6997a067b1a83:B 2024-11-08T00:24:49,230 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-08T00:24:49,231 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-08T00:24:49,231 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-08T00:24:49,231 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. because compaction request was cancelled 2024-11-08T00:24:49,231 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c140d3911f03ee9ce6997a067b1a83:C 2024-11-08T00:24:49,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-08T00:24:49,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:49,233 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 94c140d3911f03ee9ce6997a067b1a83 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-08T00:24:49,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/525ef286526f4d53bd6d0e975f91a881 is 50, key is test_row_0/A:col10/1731025488132/Put/seqid=0 2024-11-08T00:24:49,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742113_1289 (size=12915) 2024-11-08T00:24:49,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742114_1290 (size=12301) 2024-11-08T00:24:49,280 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/525ef286526f4d53bd6d0e975f91a881 2024-11-08T00:24:49,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 94c140d3911f03ee9ce6997a067b1a83 2024-11-08T00:24:49,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. as already flushing 2024-11-08T00:24:49,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/99c8524e26b44e928b8874ef89489786 is 50, key is test_row_0/B:col10/1731025488132/Put/seqid=0 2024-11-08T00:24:49,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742115_1291 (size=12301) 2024-11-08T00:24:49,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:49,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45704 deadline: 1731025549475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:49,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:49,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:49,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:49,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:49,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45704 deadline: 1731025549587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:49,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:49,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:49,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:49,663 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/2ab8a02378eb447f8bedd75dad2f87f3 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/2ab8a02378eb447f8bedd75dad2f87f3 2024-11-08T00:24:49,669 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94c140d3911f03ee9ce6997a067b1a83/A of 94c140d3911f03ee9ce6997a067b1a83 into 2ab8a02378eb447f8bedd75dad2f87f3(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:49,669 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:24:49,669 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., storeName=94c140d3911f03ee9ce6997a067b1a83/A, priority=13, startTime=1731025489137; duration=0sec 2024-11-08T00:24:49,669 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:49,669 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c140d3911f03ee9ce6997a067b1a83:A 2024-11-08T00:24:49,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-08T00:24:49,744 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/99c8524e26b44e928b8874ef89489786 2024-11-08T00:24:49,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/68c6531d0ebf46bf9291b04707fdf99b is 50, key is test_row_0/C:col10/1731025488132/Put/seqid=0 2024-11-08T00:24:49,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742116_1292 (size=12301) 2024-11-08T00:24:49,790 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/68c6531d0ebf46bf9291b04707fdf99b 2024-11-08T00:24:49,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/525ef286526f4d53bd6d0e975f91a881 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/525ef286526f4d53bd6d0e975f91a881 2024-11-08T00:24:49,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:49,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45704 deadline: 1731025549798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:49,805 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:49,805 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:49,805 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:49,806 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/525ef286526f4d53bd6d0e975f91a881, entries=150, sequenceid=290, filesize=12.0 K 2024-11-08T00:24:49,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/99c8524e26b44e928b8874ef89489786 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/99c8524e26b44e928b8874ef89489786 2024-11-08T00:24:49,812 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/99c8524e26b44e928b8874ef89489786, entries=150, sequenceid=290, filesize=12.0 K 2024-11-08T00:24:49,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/68c6531d0ebf46bf9291b04707fdf99b as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/68c6531d0ebf46bf9291b04707fdf99b 2024-11-08T00:24:49,819 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/68c6531d0ebf46bf9291b04707fdf99b, entries=150, sequenceid=290, filesize=12.0 K 2024-11-08T00:24:49,820 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 94c140d3911f03ee9ce6997a067b1a83 in 588ms, sequenceid=290, compaction requested=true 2024-11-08T00:24:49,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:24:49,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:49,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-08T00:24:49,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-08T00:24:49,824 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-08T00:24:49,825 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2510 sec 2024-11-08T00:24:49,826 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.2570 sec 2024-11-08T00:24:50,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 94c140d3911f03ee9ce6997a067b1a83 2024-11-08T00:24:50,081 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:24:50,081 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94c140d3911f03ee9ce6997a067b1a83 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-08T00:24:50,095 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/01f1031e07f4400d91504888ea614f56 is 50, key is test_row_0/A:col10/1731025489473/Put/seqid=0 2024-11-08T00:24:50,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:50,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025550093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:50,100 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:50,100 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:50,100 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:50,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:50,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025550095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:50,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:50,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:50,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:50,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:50,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025550096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:50,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:50,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:50,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:50,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:50,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025550100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:50,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:50,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:50,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:50,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:50,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45704 deadline: 1731025550117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:50,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:50,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:50,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:50,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742117_1293 (size=12301) 2024-11-08T00:24:50,149 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/01f1031e07f4400d91504888ea614f56 2024-11-08T00:24:50,189 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/457a985ce6c34bc3b793e39da3d5d421 is 50, key is test_row_0/B:col10/1731025489473/Put/seqid=0 2024-11-08T00:24:50,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:50,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025550208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:50,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:50,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:50,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025550208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:50,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025550209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:50,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:50,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025550209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:50,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:50,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:50,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:50,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:50,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:50,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:50,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:50,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:50,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:50,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:50,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:50,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:50,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742118_1294 (size=12301) 2024-11-08T00:24:50,226 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/457a985ce6c34bc3b793e39da3d5d421 2024-11-08T00:24:50,254 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/9626c3067d5240059e8ba479bb0b74af is 50, key is test_row_0/C:col10/1731025489473/Put/seqid=0 2024-11-08T00:24:50,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742119_1295 (size=12301) 2024-11-08T00:24:50,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:50,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025550417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:50,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:50,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:50,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:50,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025550417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:50,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:50,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025550417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:50,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:50,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:50,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:50,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:50,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:50,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025550417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:50,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:50,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:50,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:50,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:50,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:50,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:50,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:50,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45704 deadline: 1731025550627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:50,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:50,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:50,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:50,686 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/9626c3067d5240059e8ba479bb0b74af 2024-11-08T00:24:50,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/01f1031e07f4400d91504888ea614f56 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/01f1031e07f4400d91504888ea614f56 2024-11-08T00:24:50,709 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/01f1031e07f4400d91504888ea614f56, entries=150, sequenceid=317, filesize=12.0 K 2024-11-08T00:24:50,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/457a985ce6c34bc3b793e39da3d5d421 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/457a985ce6c34bc3b793e39da3d5d421 2024-11-08T00:24:50,716 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/457a985ce6c34bc3b793e39da3d5d421, entries=150, sequenceid=317, filesize=12.0 K 2024-11-08T00:24:50,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/9626c3067d5240059e8ba479bb0b74af as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/9626c3067d5240059e8ba479bb0b74af 2024-11-08T00:24:50,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-08T00:24:50,718 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:24:50,718 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-08T00:24:50,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-08T00:24:50,721 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T00:24:50,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-08T00:24:50,721 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T00:24:50,722 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T00:24:50,727 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/9626c3067d5240059e8ba479bb0b74af, entries=150, sequenceid=317, filesize=12.0 K 2024-11-08T00:24:50,729 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 94c140d3911f03ee9ce6997a067b1a83 in 648ms, sequenceid=317, compaction requested=true 2024-11-08T00:24:50,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:24:50,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c140d3911f03ee9ce6997a067b1a83:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T00:24:50,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:24:50,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c140d3911f03ee9ce6997a067b1a83:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T00:24:50,729 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:24:50,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:24:50,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c140d3911f03ee9ce6997a067b1a83:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:24:50,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-08T00:24:50,731 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:24:50,731 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): 94c140d3911f03ee9ce6997a067b1a83/A is initiating minor compaction (all files) 2024-11-08T00:24:50,731 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c140d3911f03ee9ce6997a067b1a83/A in TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:50,732 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/2ab8a02378eb447f8bedd75dad2f87f3, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/525ef286526f4d53bd6d0e975f91a881, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/01f1031e07f4400d91504888ea614f56] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp, totalSize=36.6 K 2024-11-08T00:24:50,733 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ab8a02378eb447f8bedd75dad2f87f3, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731025487907 2024-11-08T00:24:50,733 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T00:24:50,733 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 525ef286526f4d53bd6d0e975f91a881, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731025488125 2024-11-08T00:24:50,734 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01f1031e07f4400d91504888ea614f56, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731025489473 2024-11-08T00:24:50,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 94c140d3911f03ee9ce6997a067b1a83 2024-11-08T00:24:50,736 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49600 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T00:24:50,736 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): 94c140d3911f03ee9ce6997a067b1a83/C is initiating minor compaction (all files) 2024-11-08T00:24:50,736 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c140d3911f03ee9ce6997a067b1a83/C in TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:50,736 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/0af7a379dd804196ade4bf97a9ff1686, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/49fd83d260624c0585fe344a52bd09cf, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/68c6531d0ebf46bf9291b04707fdf99b, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/9626c3067d5240059e8ba479bb0b74af] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp, totalSize=48.4 K 2024-11-08T00:24:50,737 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 0af7a379dd804196ade4bf97a9ff1686, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731025487743 2024-11-08T00:24:50,738 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 49fd83d260624c0585fe344a52bd09cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731025487907 2024-11-08T00:24:50,738 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:24:50,738 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94c140d3911f03ee9ce6997a067b1a83 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-08T00:24:50,739 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 68c6531d0ebf46bf9291b04707fdf99b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731025488125 2024-11-08T00:24:50,741 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 9626c3067d5240059e8ba479bb0b74af, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731025489473 2024-11-08T00:24:50,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/581a8d2cac7949c883d7c81731c7ec36 is 50, key is test_row_0/A:col10/1731025490092/Put/seqid=0 2024-11-08T00:24:50,758 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c140d3911f03ee9ce6997a067b1a83#A#compaction#249 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:50,759 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/359eed3ddd7c41a28051ae7d3f09f79d is 50, key is test_row_0/A:col10/1731025489473/Put/seqid=0 2024-11-08T00:24:50,782 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c140d3911f03ee9ce6997a067b1a83#C#compaction#250 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:50,783 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/60e0fdef436042c184a9a89cd1838419 is 50, key is test_row_0/C:col10/1731025489473/Put/seqid=0 2024-11-08T00:24:50,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742120_1296 (size=12301) 2024-11-08T00:24:50,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:50,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025550802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:50,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:50,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:50,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:50,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:50,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025550802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:50,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:50,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:50,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:50,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:50,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025550803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:50,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:50,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:50,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:50,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:50,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025550803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:50,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:50,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:50,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:50,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742121_1297 (size=13017) 2024-11-08T00:24:50,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742122_1298 (size=12983) 2024-11-08T00:24:50,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-08T00:24:50,830 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/60e0fdef436042c184a9a89cd1838419 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/60e0fdef436042c184a9a89cd1838419 2024-11-08T00:24:50,840 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/359eed3ddd7c41a28051ae7d3f09f79d as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/359eed3ddd7c41a28051ae7d3f09f79d 2024-11-08T00:24:50,842 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 94c140d3911f03ee9ce6997a067b1a83/C of 94c140d3911f03ee9ce6997a067b1a83 into 60e0fdef436042c184a9a89cd1838419(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:50,842 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:24:50,842 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., storeName=94c140d3911f03ee9ce6997a067b1a83/C, priority=12, startTime=1731025490729; duration=0sec 2024-11-08T00:24:50,842 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:24:50,842 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c140d3911f03ee9ce6997a067b1a83:C 2024-11-08T00:24:50,842 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:24:50,844 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:24:50,844 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): 94c140d3911f03ee9ce6997a067b1a83/B is initiating minor compaction (all files) 2024-11-08T00:24:50,844 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c140d3911f03ee9ce6997a067b1a83/B in TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:50,845 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/2b7a9e9d2ca5447ea13eb39151206822, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/99c8524e26b44e928b8874ef89489786, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/457a985ce6c34bc3b793e39da3d5d421] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp, totalSize=36.6 K 2024-11-08T00:24:50,845 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b7a9e9d2ca5447ea13eb39151206822, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731025487907 2024-11-08T00:24:50,846 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 99c8524e26b44e928b8874ef89489786, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731025488125 2024-11-08T00:24:50,847 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 457a985ce6c34bc3b793e39da3d5d421, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731025489473 2024-11-08T00:24:50,847 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94c140d3911f03ee9ce6997a067b1a83/A of 94c140d3911f03ee9ce6997a067b1a83 into 359eed3ddd7c41a28051ae7d3f09f79d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:50,847 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:24:50,847 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., storeName=94c140d3911f03ee9ce6997a067b1a83/A, priority=13, startTime=1731025490729; duration=0sec 2024-11-08T00:24:50,847 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:50,847 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c140d3911f03ee9ce6997a067b1a83:A 2024-11-08T00:24:50,874 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c140d3911f03ee9ce6997a067b1a83#B#compaction#251 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:50,875 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/008cbdaba15544258a40ee6966bb01e9 is 50, key is test_row_0/B:col10/1731025489473/Put/seqid=0 2024-11-08T00:24:50,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-08T00:24:50,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:50,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. as already flushing 2024-11-08T00:24:50,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:50,877 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:50,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:50,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:50,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742123_1299 (size=13017) 2024-11-08T00:24:50,895 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/008cbdaba15544258a40ee6966bb01e9 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/008cbdaba15544258a40ee6966bb01e9 2024-11-08T00:24:50,904 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94c140d3911f03ee9ce6997a067b1a83/B of 94c140d3911f03ee9ce6997a067b1a83 into 008cbdaba15544258a40ee6966bb01e9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:50,904 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:24:50,904 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., storeName=94c140d3911f03ee9ce6997a067b1a83/B, priority=13, startTime=1731025490729; duration=0sec 2024-11-08T00:24:50,905 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:50,905 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c140d3911f03ee9ce6997a067b1a83:B 2024-11-08T00:24:50,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:50,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025550917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:50,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:50,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:50,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:50,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025550917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:50,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:50,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:50,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:50,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:50,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:50,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025550917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:50,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:50,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025550917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:50,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:50,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:50,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:50,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:50,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:50,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:51,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-08T00:24:51,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:51,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. as already flushing 2024-11-08T00:24:51,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:51,030 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:51,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:51,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:51,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-08T00:24:51,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:51,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025551128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:51,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:51,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:51,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:51,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:51,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025551128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:51,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:51,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:51,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:51,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:51,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025551128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:51,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:51,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025551129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:51,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:51,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:51,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:51,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:51,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:51,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:51,181 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-08T00:24:51,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:51,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. as already flushing 2024-11-08T00:24:51,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:51,182 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:51,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:51,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:51,187 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/581a8d2cac7949c883d7c81731c7ec36 2024-11-08T00:24:51,195 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/8bf9246939b946ae9e2b15aceb15c0a1 is 50, key is test_row_0/B:col10/1731025490092/Put/seqid=0 2024-11-08T00:24:51,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742124_1300 (size=12301) 2024-11-08T00:24:51,200 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/8bf9246939b946ae9e2b15aceb15c0a1 2024-11-08T00:24:51,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/523fcae6f9a64ee7a5067dc91d5f04c1 is 50, key is test_row_0/C:col10/1731025490092/Put/seqid=0 2024-11-08T00:24:51,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742125_1301 (size=12301) 2024-11-08T00:24:51,214 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/523fcae6f9a64ee7a5067dc91d5f04c1 2024-11-08T00:24:51,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/581a8d2cac7949c883d7c81731c7ec36 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/581a8d2cac7949c883d7c81731c7ec36 2024-11-08T00:24:51,229 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/581a8d2cac7949c883d7c81731c7ec36, entries=150, sequenceid=330, filesize=12.0 K 2024-11-08T00:24:51,231 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/8bf9246939b946ae9e2b15aceb15c0a1 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/8bf9246939b946ae9e2b15aceb15c0a1 2024-11-08T00:24:51,238 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/8bf9246939b946ae9e2b15aceb15c0a1, entries=150, sequenceid=330, filesize=12.0 K 2024-11-08T00:24:51,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/523fcae6f9a64ee7a5067dc91d5f04c1 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/523fcae6f9a64ee7a5067dc91d5f04c1 2024-11-08T00:24:51,261 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/523fcae6f9a64ee7a5067dc91d5f04c1, entries=150, sequenceid=330, filesize=12.0 K 2024-11-08T00:24:51,262 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 94c140d3911f03ee9ce6997a067b1a83 in 524ms, sequenceid=330, compaction requested=false 2024-11-08T00:24:51,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:24:51,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-08T00:24:51,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:51,337 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 94c140d3911f03ee9ce6997a067b1a83 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-08T00:24:51,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/22bd69856e984e16b9e96ed452ba2d44 is 50, key is test_row_0/A:col10/1731025490795/Put/seqid=0 2024-11-08T00:24:51,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-08T00:24:51,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742126_1302 (size=12301) 2024-11-08T00:24:51,439 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. as already flushing 2024-11-08T00:24:51,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 94c140d3911f03ee9ce6997a067b1a83 2024-11-08T00:24:51,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:51,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025551446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:51,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:51,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025551448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:51,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:51,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:51,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:51,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:51,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:51,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:51,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:51,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025551449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:51,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:51,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:51,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:51,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:51,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025551452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:51,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:51,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:51,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:51,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:51,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025551558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:51,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:51,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:51,560 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:51,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:51,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025551558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:51,561 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:51,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:51,561 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:51,561 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:51,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025551558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:51,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:51,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:51,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:51,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:51,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025551567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:51,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:51,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:51,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:51,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:51,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45704 deadline: 1731025551647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:51,649 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 3302f0f507bd,40479,1731025415856 for 'test_row_2' in 94c140d3911f03ee9ce6997a067b1a83 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T00:24:51,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:51,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:51,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:51,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:51,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025551768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:51,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:51,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:51,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:51,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025551768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:51,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:51,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:51,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025551768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:51,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:51,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:51,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:51,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:51,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:51,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:51,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:51,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025551777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:51,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:51,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:51,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:51,784 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/22bd69856e984e16b9e96ed452ba2d44 2024-11-08T00:24:51,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/1bd56d42bf44433d84024cf406a31b31 is 50, key is test_row_0/B:col10/1731025490795/Put/seqid=0 2024-11-08T00:24:51,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742127_1303 (size=12301) 2024-11-08T00:24:51,803 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/1bd56d42bf44433d84024cf406a31b31 2024-11-08T00:24:51,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/6678a2a199a948e499d663c93ab95496 is 50, key is test_row_0/C:col10/1731025490795/Put/seqid=0 2024-11-08T00:24:51,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742128_1304 (size=12301) 2024-11-08T00:24:51,839 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/6678a2a199a948e499d663c93ab95496 2024-11-08T00:24:51,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/22bd69856e984e16b9e96ed452ba2d44 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/22bd69856e984e16b9e96ed452ba2d44 2024-11-08T00:24:51,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-08T00:24:51,861 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/22bd69856e984e16b9e96ed452ba2d44, entries=150, sequenceid=357, filesize=12.0 K 2024-11-08T00:24:51,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/1bd56d42bf44433d84024cf406a31b31 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/1bd56d42bf44433d84024cf406a31b31 2024-11-08T00:24:51,868 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/1bd56d42bf44433d84024cf406a31b31, entries=150, sequenceid=357, filesize=12.0 K 2024-11-08T00:24:51,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/6678a2a199a948e499d663c93ab95496 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/6678a2a199a948e499d663c93ab95496 2024-11-08T00:24:51,878 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/6678a2a199a948e499d663c93ab95496, entries=150, sequenceid=357, filesize=12.0 K 2024-11-08T00:24:51,880 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 94c140d3911f03ee9ce6997a067b1a83 in 542ms, sequenceid=357, compaction requested=true 2024-11-08T00:24:51,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:24:51,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:51,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-08T00:24:51,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-08T00:24:51,883 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-08T00:24:51,883 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1590 sec 2024-11-08T00:24:51,886 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.1650 sec 2024-11-08T00:24:52,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 94c140d3911f03ee9ce6997a067b1a83 2024-11-08T00:24:52,081 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:24:52,081 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94c140d3911f03ee9ce6997a067b1a83 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-08T00:24:52,109 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/40776fa00c0144c0b471cb0008ce4b4f is 50, key is test_row_0/A:col10/1731025491442/Put/seqid=0 2024-11-08T00:24:52,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742129_1305 (size=14741) 2024-11-08T00:24:52,118 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/40776fa00c0144c0b471cb0008ce4b4f 2024-11-08T00:24:52,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/ae0c17d2e053464ba6410bbd6d98d6ba is 50, key is test_row_0/B:col10/1731025491442/Put/seqid=0 2024-11-08T00:24:52,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:52,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:52,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025552153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:52,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025552152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:52,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:52,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:52,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:52,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:52,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:52,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:52,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:52,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025552159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:52,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:52,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:52,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:52,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:52,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025552160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:52,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:52,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:52,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:52,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742130_1306 (size=12301) 2024-11-08T00:24:52,190 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/ae0c17d2e053464ba6410bbd6d98d6ba 2024-11-08T00:24:52,213 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/3124bd10656c436f963b8e0c6a207f37 is 50, key is test_row_0/C:col10/1731025491442/Put/seqid=0 2024-11-08T00:24:52,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742131_1307 (size=12301) 2024-11-08T00:24:52,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:52,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025552268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:52,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:52,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:52,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025552268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:52,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:52,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:52,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:52,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025552268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:52,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:52,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:52,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:52,275 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:52,275 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:52,275 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:52,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:52,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025552277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:52,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:52,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:52,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:52,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:52,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025552478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:52,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:52,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025552478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:52,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:52,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:52,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:52,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:52,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:52,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:52,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:52,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025552478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:52,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:52,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:52,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:52,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:52,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025552488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:52,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:52,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:52,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:52,623 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/3124bd10656c436f963b8e0c6a207f37 2024-11-08T00:24:52,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/40776fa00c0144c0b471cb0008ce4b4f as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/40776fa00c0144c0b471cb0008ce4b4f 2024-11-08T00:24:52,641 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/40776fa00c0144c0b471cb0008ce4b4f, entries=200, sequenceid=370, filesize=14.4 K 2024-11-08T00:24:52,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/ae0c17d2e053464ba6410bbd6d98d6ba as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/ae0c17d2e053464ba6410bbd6d98d6ba 2024-11-08T00:24:52,649 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/ae0c17d2e053464ba6410bbd6d98d6ba, entries=150, sequenceid=370, filesize=12.0 K 2024-11-08T00:24:52,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/3124bd10656c436f963b8e0c6a207f37 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/3124bd10656c436f963b8e0c6a207f37 2024-11-08T00:24:52,654 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/3124bd10656c436f963b8e0c6a207f37, entries=150, sequenceid=370, filesize=12.0 K 2024-11-08T00:24:52,655 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 94c140d3911f03ee9ce6997a067b1a83 in 574ms, sequenceid=370, compaction requested=true 2024-11-08T00:24:52,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:24:52,655 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T00:24:52,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c140d3911f03ee9ce6997a067b1a83:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T00:24:52,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:52,656 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T00:24:52,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c140d3911f03ee9ce6997a067b1a83:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T00:24:52,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:52,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c140d3911f03ee9ce6997a067b1a83:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:24:52,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:24:52,657 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T00:24:52,657 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): 94c140d3911f03ee9ce6997a067b1a83/A is initiating minor compaction (all files) 2024-11-08T00:24:52,657 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c140d3911f03ee9ce6997a067b1a83/A in TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:52,657 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/359eed3ddd7c41a28051ae7d3f09f79d, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/581a8d2cac7949c883d7c81731c7ec36, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/22bd69856e984e16b9e96ed452ba2d44, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/40776fa00c0144c0b471cb0008ce4b4f] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp, totalSize=51.1 K 2024-11-08T00:24:52,658 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T00:24:52,658 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): 94c140d3911f03ee9ce6997a067b1a83/B is initiating minor compaction (all files) 2024-11-08T00:24:52,658 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c140d3911f03ee9ce6997a067b1a83/B in TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:52,658 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/008cbdaba15544258a40ee6966bb01e9, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/8bf9246939b946ae9e2b15aceb15c0a1, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/1bd56d42bf44433d84024cf406a31b31, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/ae0c17d2e053464ba6410bbd6d98d6ba] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp, totalSize=48.8 K 2024-11-08T00:24:52,658 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 359eed3ddd7c41a28051ae7d3f09f79d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731025489473 2024-11-08T00:24:52,659 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 008cbdaba15544258a40ee6966bb01e9, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731025489473 2024-11-08T00:24:52,659 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 581a8d2cac7949c883d7c81731c7ec36, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731025490092 2024-11-08T00:24:52,659 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 8bf9246939b946ae9e2b15aceb15c0a1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731025490092 2024-11-08T00:24:52,659 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22bd69856e984e16b9e96ed452ba2d44, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731025490795 2024-11-08T00:24:52,660 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bd56d42bf44433d84024cf406a31b31, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731025490795 2024-11-08T00:24:52,660 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40776fa00c0144c0b471cb0008ce4b4f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731025491442 2024-11-08T00:24:52,660 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting ae0c17d2e053464ba6410bbd6d98d6ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731025491442 2024-11-08T00:24:52,671 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c140d3911f03ee9ce6997a067b1a83#B#compaction#260 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:52,672 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/be4830226c41488b989bcfcaa60888c8 is 50, key is test_row_0/B:col10/1731025491442/Put/seqid=0 2024-11-08T00:24:52,675 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c140d3911f03ee9ce6997a067b1a83#A#compaction#261 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:52,675 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/5b00d2154e064af29355c56ec9fc9dd5 is 50, key is test_row_0/A:col10/1731025491442/Put/seqid=0 2024-11-08T00:24:52,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742132_1308 (size=13153) 2024-11-08T00:24:52,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742133_1309 (size=13153) 2024-11-08T00:24:52,694 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/be4830226c41488b989bcfcaa60888c8 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/be4830226c41488b989bcfcaa60888c8 2024-11-08T00:24:52,709 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/5b00d2154e064af29355c56ec9fc9dd5 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/5b00d2154e064af29355c56ec9fc9dd5 2024-11-08T00:24:52,710 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 94c140d3911f03ee9ce6997a067b1a83/B of 94c140d3911f03ee9ce6997a067b1a83 into be4830226c41488b989bcfcaa60888c8(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:52,710 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:24:52,710 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., storeName=94c140d3911f03ee9ce6997a067b1a83/B, priority=12, startTime=1731025492656; duration=0sec 2024-11-08T00:24:52,712 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:24:52,712 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c140d3911f03ee9ce6997a067b1a83:B 2024-11-08T00:24:52,712 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T00:24:52,714 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T00:24:52,715 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): 94c140d3911f03ee9ce6997a067b1a83/C is initiating minor compaction (all files) 2024-11-08T00:24:52,715 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c140d3911f03ee9ce6997a067b1a83/C in TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:52,715 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/60e0fdef436042c184a9a89cd1838419, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/523fcae6f9a64ee7a5067dc91d5f04c1, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/6678a2a199a948e499d663c93ab95496, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/3124bd10656c436f963b8e0c6a207f37] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp, totalSize=48.7 K 2024-11-08T00:24:52,716 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 60e0fdef436042c184a9a89cd1838419, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731025489473 2024-11-08T00:24:52,718 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 94c140d3911f03ee9ce6997a067b1a83/A of 94c140d3911f03ee9ce6997a067b1a83 into 5b00d2154e064af29355c56ec9fc9dd5(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:52,718 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 523fcae6f9a64ee7a5067dc91d5f04c1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731025490092 2024-11-08T00:24:52,718 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:24:52,718 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., storeName=94c140d3911f03ee9ce6997a067b1a83/A, priority=12, startTime=1731025492655; duration=0sec 2024-11-08T00:24:52,718 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:52,718 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c140d3911f03ee9ce6997a067b1a83:A 2024-11-08T00:24:52,719 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 6678a2a199a948e499d663c93ab95496, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731025490795 2024-11-08T00:24:52,719 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 3124bd10656c436f963b8e0c6a207f37, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731025491442 2024-11-08T00:24:52,729 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c140d3911f03ee9ce6997a067b1a83#C#compaction#262 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:52,729 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/49e98abd406045d583b000d028570898 is 50, key is test_row_0/C:col10/1731025491442/Put/seqid=0 2024-11-08T00:24:52,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742134_1310 (size=13119) 2024-11-08T00:24:52,791 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:24:52,792 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94c140d3911f03ee9ce6997a067b1a83 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-08T00:24:52,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 94c140d3911f03ee9ce6997a067b1a83 2024-11-08T00:24:52,801 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/3c6adb5cdb614f7b9283affd8a1493ec is 50, key is test_row_0/A:col10/1731025492157/Put/seqid=0 2024-11-08T00:24:52,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:52,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025552808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:52,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:52,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:52,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:52,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:52,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025552809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:52,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:52,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:52,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:52,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:52,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025552813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:52,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:52,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:52,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:52,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:52,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025552816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:52,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:52,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:52,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:52,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742135_1311 (size=14741) 2024-11-08T00:24:52,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-08T00:24:52,868 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:24:52,868 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-08T00:24:52,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-08T00:24:52,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-08T00:24:52,871 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T00:24:52,872 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T00:24:52,872 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T00:24:52,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:52,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025552917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:52,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:52,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:52,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:52,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:52,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025552927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:52,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:52,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:52,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:52,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:52,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025552927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:52,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:52,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:52,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:52,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:52,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025552927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:52,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:52,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:52,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:52,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-08T00:24:53,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-08T00:24:53,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:53,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. as already flushing 2024-11-08T00:24:53,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:53,025 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:53,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:53,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:53,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:53,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025553127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:53,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:53,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:53,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:53,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:53,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025553137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:53,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:53,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:53,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:53,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:53,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025553137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:53,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:53,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:53,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:53,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:53,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025553147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:53,151 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:53,151 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:53,151 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:53,179 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/49e98abd406045d583b000d028570898 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/49e98abd406045d583b000d028570898 2024-11-08T00:24:53,180 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-08T00:24:53,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:53,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. as already flushing 2024-11-08T00:24:53,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:53,181 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:53,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:53,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:53,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-08T00:24:53,189 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 94c140d3911f03ee9ce6997a067b1a83/C of 94c140d3911f03ee9ce6997a067b1a83 into 49e98abd406045d583b000d028570898(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:53,189 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:24:53,189 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., storeName=94c140d3911f03ee9ce6997a067b1a83/C, priority=12, startTime=1731025492656; duration=0sec 2024-11-08T00:24:53,189 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:53,189 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c140d3911f03ee9ce6997a067b1a83:C 2024-11-08T00:24:53,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/3c6adb5cdb614f7b9283affd8a1493ec 2024-11-08T00:24:53,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/18a1a997cc724d4aa78b82de804f58ac is 50, key is test_row_0/B:col10/1731025492157/Put/seqid=0 2024-11-08T00:24:53,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742136_1312 (size=12301) 2024-11-08T00:24:53,333 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-08T00:24:53,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:53,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. as already flushing 2024-11-08T00:24:53,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:53,334 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:53,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:53,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:53,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:53,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025553437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:53,438 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:53,438 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:53,438 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:53,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:53,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025553447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:53,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:53,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:53,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025553447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:53,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:53,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:53,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:53,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:53,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:53,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:53,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025553458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:53,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:53,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:53,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:53,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-08T00:24:53,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:53,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. as already flushing 2024-11-08T00:24:53,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:53,488 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:53,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:53,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:53,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-08T00:24:53,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-08T00:24:53,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:53,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. as already flushing 2024-11-08T00:24:53,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:53,643 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:53,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:53,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:53,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:53,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45704 deadline: 1731025553657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:53,660 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 3302f0f507bd,40479,1731025415856 for 'test_row_2' in 94c140d3911f03ee9ce6997a067b1a83 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T00:24:53,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:53,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:53,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:53,702 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/18a1a997cc724d4aa78b82de804f58ac 2024-11-08T00:24:53,730 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/ab7ff9552c644ba1b10f0af075cd19de is 50, key is test_row_0/C:col10/1731025492157/Put/seqid=0 2024-11-08T00:24:53,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742137_1313 (size=12301) 2024-11-08T00:24:53,769 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/ab7ff9552c644ba1b10f0af075cd19de 2024-11-08T00:24:53,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/3c6adb5cdb614f7b9283affd8a1493ec as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/3c6adb5cdb614f7b9283affd8a1493ec 2024-11-08T00:24:53,785 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/3c6adb5cdb614f7b9283affd8a1493ec, entries=200, sequenceid=397, filesize=14.4 K 2024-11-08T00:24:53,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/18a1a997cc724d4aa78b82de804f58ac as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/18a1a997cc724d4aa78b82de804f58ac 2024-11-08T00:24:53,791 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/18a1a997cc724d4aa78b82de804f58ac, entries=150, sequenceid=397, filesize=12.0 K 2024-11-08T00:24:53,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/ab7ff9552c644ba1b10f0af075cd19de as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/ab7ff9552c644ba1b10f0af075cd19de 2024-11-08T00:24:53,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-08T00:24:53,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:53,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. as already flushing 2024-11-08T00:24:53,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:53,797 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:53,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:53,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:53,804 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/ab7ff9552c644ba1b10f0af075cd19de, entries=150, sequenceid=397, filesize=12.0 K 2024-11-08T00:24:53,805 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 94c140d3911f03ee9ce6997a067b1a83 in 1014ms, sequenceid=397, compaction requested=false 2024-11-08T00:24:53,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:24:53,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 94c140d3911f03ee9ce6997a067b1a83 2024-11-08T00:24:53,951 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:24:53,951 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94c140d3911f03ee9ce6997a067b1a83 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-08T00:24:53,953 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-08T00:24:53,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:53,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. as already flushing 2024-11-08T00:24:53,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:53,953 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:53,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:53,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:53,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/7bba2f50971e42b7a896a04a997f090c is 50, key is test_row_0/A:col10/1731025492812/Put/seqid=0 2024-11-08T00:24:53,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742138_1314 (size=12301) 2024-11-08T00:24:53,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:53,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025553990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:53,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:53,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:53,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:53,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:53,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025553991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:53,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:53,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:53,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:53,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025553993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:53,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:53,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:53,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:53,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:54,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:54,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025553995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:54,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:54,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:54,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:54,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-08T00:24:54,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:54,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025554097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:54,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:54,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:54,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:54,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-08T00:24:54,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:54,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. as already flushing 2024-11-08T00:24:54,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:54,109 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:54,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:54,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:54,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:54,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025554108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:54,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:54,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025554108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:54,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:54,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:54,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:54,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:54,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:54,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:54,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:54,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025554116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:54,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:54,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:54,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:54,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-08T00:24:54,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:54,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. as already flushing 2024-11-08T00:24:54,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:54,262 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:54,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:54,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:54,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:54,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025554307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:54,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:54,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:54,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:54,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:54,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025554319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:54,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:54,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:54,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:54,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:54,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025554319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:54,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:54,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:54,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:54,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:54,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025554329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:54,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:54,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:54,333 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:54,369 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/7bba2f50971e42b7a896a04a997f090c 2024-11-08T00:24:54,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/672f6bb0d7594c9285311d5bcf19b0a0 is 50, key is test_row_0/B:col10/1731025492812/Put/seqid=0 2024-11-08T00:24:54,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-08T00:24:54,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:54,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. as already flushing 2024-11-08T00:24:54,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:54,419 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:54,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:54,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:54,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742139_1315 (size=12301) 2024-11-08T00:24:54,430 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/672f6bb0d7594c9285311d5bcf19b0a0 2024-11-08T00:24:54,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/5dd310ccc61d41f480154c9fbe7c8a0f is 50, key is test_row_0/C:col10/1731025492812/Put/seqid=0 2024-11-08T00:24:54,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742140_1316 (size=12301) 2024-11-08T00:24:54,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-08T00:24:54,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:54,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. as already flushing 2024-11-08T00:24:54,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:54,572 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:54,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:54,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:54,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:54,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025554618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:54,621 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:54,621 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:54,621 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:54,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:54,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025554627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:54,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:54,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025554627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:54,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:54,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:54,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:54,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:54,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:54,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:54,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:54,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025554637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:54,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:54,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:54,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:54,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-08T00:24:54,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:54,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. as already flushing 2024-11-08T00:24:54,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:54,726 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:54,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:54,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:54,851 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/5dd310ccc61d41f480154c9fbe7c8a0f 2024-11-08T00:24:54,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/7bba2f50971e42b7a896a04a997f090c as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/7bba2f50971e42b7a896a04a997f090c 2024-11-08T00:24:54,859 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/7bba2f50971e42b7a896a04a997f090c, entries=150, sequenceid=411, filesize=12.0 K 2024-11-08T00:24:54,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/672f6bb0d7594c9285311d5bcf19b0a0 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/672f6bb0d7594c9285311d5bcf19b0a0 2024-11-08T00:24:54,865 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/672f6bb0d7594c9285311d5bcf19b0a0, entries=150, sequenceid=411, filesize=12.0 K 2024-11-08T00:24:54,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/5dd310ccc61d41f480154c9fbe7c8a0f as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/5dd310ccc61d41f480154c9fbe7c8a0f 2024-11-08T00:24:54,875 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/5dd310ccc61d41f480154c9fbe7c8a0f, entries=150, sequenceid=411, filesize=12.0 K 2024-11-08T00:24:54,876 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 94c140d3911f03ee9ce6997a067b1a83 in 924ms, sequenceid=411, compaction requested=true 2024-11-08T00:24:54,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:24:54,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c140d3911f03ee9ce6997a067b1a83:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T00:24:54,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:54,876 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:24:54,876 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:24:54,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c140d3911f03ee9ce6997a067b1a83:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T00:24:54,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:54,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c140d3911f03ee9ce6997a067b1a83:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:24:54,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:24:54,879 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:24:54,879 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:24:54,879 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): 94c140d3911f03ee9ce6997a067b1a83/B is initiating minor compaction (all files) 2024-11-08T00:24:54,879 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): 94c140d3911f03ee9ce6997a067b1a83/A is initiating minor compaction (all files) 2024-11-08T00:24:54,880 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c140d3911f03ee9ce6997a067b1a83/B in TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:54,880 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c140d3911f03ee9ce6997a067b1a83/A in TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:54,880 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/be4830226c41488b989bcfcaa60888c8, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/18a1a997cc724d4aa78b82de804f58ac, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/672f6bb0d7594c9285311d5bcf19b0a0] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp, totalSize=36.9 K 2024-11-08T00:24:54,880 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/5b00d2154e064af29355c56ec9fc9dd5, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/3c6adb5cdb614f7b9283affd8a1493ec, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/7bba2f50971e42b7a896a04a997f090c] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp, totalSize=39.3 K 2024-11-08T00:24:54,880 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-08T00:24:54,881 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b00d2154e064af29355c56ec9fc9dd5, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731025491442 2024-11-08T00:24:54,881 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting be4830226c41488b989bcfcaa60888c8, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731025491442 2024-11-08T00:24:54,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:54,882 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 94c140d3911f03ee9ce6997a067b1a83 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-08T00:24:54,883 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c6adb5cdb614f7b9283affd8a1493ec, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1731025492142 2024-11-08T00:24:54,883 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 18a1a997cc724d4aa78b82de804f58ac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1731025492142 2024-11-08T00:24:54,884 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 672f6bb0d7594c9285311d5bcf19b0a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1731025492811 2024-11-08T00:24:54,884 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7bba2f50971e42b7a896a04a997f090c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1731025492811 2024-11-08T00:24:54,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/beb16b12b92b41f7be8321be4d12e20e is 50, key is test_row_0/A:col10/1731025493992/Put/seqid=0 2024-11-08T00:24:54,905 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c140d3911f03ee9ce6997a067b1a83#A#compaction#270 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:54,906 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/60111580c7f545e5a5c29d464e7e7324 is 50, key is test_row_0/A:col10/1731025492812/Put/seqid=0 2024-11-08T00:24:54,914 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c140d3911f03ee9ce6997a067b1a83#B#compaction#271 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:54,915 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/b45e45c889ca45c1bf14521a41adbe43 is 50, key is test_row_0/B:col10/1731025492812/Put/seqid=0 2024-11-08T00:24:54,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742141_1317 (size=12301) 2024-11-08T00:24:54,947 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/beb16b12b92b41f7be8321be4d12e20e 2024-11-08T00:24:54,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742142_1318 (size=13255) 2024-11-08T00:24:54,967 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/60111580c7f545e5a5c29d464e7e7324 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/60111580c7f545e5a5c29d464e7e7324 2024-11-08T00:24:54,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/be27ea9bd5b846f8b2cd682caba7beb6 is 50, key is test_row_0/B:col10/1731025493992/Put/seqid=0 2024-11-08T00:24:54,973 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94c140d3911f03ee9ce6997a067b1a83/A of 94c140d3911f03ee9ce6997a067b1a83 into 60111580c7f545e5a5c29d464e7e7324(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:54,973 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:24:54,973 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., storeName=94c140d3911f03ee9ce6997a067b1a83/A, priority=13, startTime=1731025494876; duration=0sec 2024-11-08T00:24:54,973 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:24:54,973 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c140d3911f03ee9ce6997a067b1a83:A 2024-11-08T00:24:54,973 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:24:54,974 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:24:54,974 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): 94c140d3911f03ee9ce6997a067b1a83/C is initiating minor compaction (all files) 2024-11-08T00:24:54,974 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c140d3911f03ee9ce6997a067b1a83/C in TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:54,974 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/49e98abd406045d583b000d028570898, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/ab7ff9552c644ba1b10f0af075cd19de, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/5dd310ccc61d41f480154c9fbe7c8a0f] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp, totalSize=36.8 K 2024-11-08T00:24:54,975 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49e98abd406045d583b000d028570898, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731025491442 2024-11-08T00:24:54,975 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab7ff9552c644ba1b10f0af075cd19de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1731025492142 2024-11-08T00:24:54,976 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5dd310ccc61d41f480154c9fbe7c8a0f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1731025492811 2024-11-08T00:24:54,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742143_1319 (size=13255) 2024-11-08T00:24:55,001 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/b45e45c889ca45c1bf14521a41adbe43 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/b45e45c889ca45c1bf14521a41adbe43 2024-11-08T00:24:55,005 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c140d3911f03ee9ce6997a067b1a83#C#compaction#273 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:55,005 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/b3bfd60cbc764057ac2dfce140533c21 is 50, key is test_row_0/C:col10/1731025492812/Put/seqid=0 2024-11-08T00:24:55,013 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94c140d3911f03ee9ce6997a067b1a83/B of 94c140d3911f03ee9ce6997a067b1a83 into b45e45c889ca45c1bf14521a41adbe43(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:55,013 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:24:55,013 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., storeName=94c140d3911f03ee9ce6997a067b1a83/B, priority=13, startTime=1731025494876; duration=0sec 2024-11-08T00:24:55,013 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:55,013 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c140d3911f03ee9ce6997a067b1a83:B 2024-11-08T00:24:55,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-08T00:24:55,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742144_1320 (size=12301) 2024-11-08T00:24:55,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742145_1321 (size=13221) 2024-11-08T00:24:55,047 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/b3bfd60cbc764057ac2dfce140533c21 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/b3bfd60cbc764057ac2dfce140533c21 2024-11-08T00:24:55,084 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94c140d3911f03ee9ce6997a067b1a83/C of 94c140d3911f03ee9ce6997a067b1a83 into b3bfd60cbc764057ac2dfce140533c21(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:55,084 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:24:55,085 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., storeName=94c140d3911f03ee9ce6997a067b1a83/C, priority=13, startTime=1731025494876; duration=0sec 2024-11-08T00:24:55,085 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:55,085 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c140d3911f03ee9ce6997a067b1a83:C 2024-11-08T00:24:55,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 94c140d3911f03ee9ce6997a067b1a83 2024-11-08T00:24:55,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. as already flushing 2024-11-08T00:24:55,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:55,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025555151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:55,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:55,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:55,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:55,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:55,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025555163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:55,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:55,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:55,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:55,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:55,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025555175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:55,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:55,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:55,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025555176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:55,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:55,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:55,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:55,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:55,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:55,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:55,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025555278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:55,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:55,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:55,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:55,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:55,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025555287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:55,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:55,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025555287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:55,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:55,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:55,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:55,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:55,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025555287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:55,291 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:55,291 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:55,291 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:55,291 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:55,291 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:55,291 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:55,423 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/be27ea9bd5b846f8b2cd682caba7beb6 2024-11-08T00:24:55,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/0808968364b04bedbfa6b574723184de is 50, key is test_row_0/C:col10/1731025493992/Put/seqid=0 2024-11-08T00:24:55,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742146_1322 (size=12301) 2024-11-08T00:24:55,445 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/0808968364b04bedbfa6b574723184de 2024-11-08T00:24:55,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/beb16b12b92b41f7be8321be4d12e20e as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/beb16b12b92b41f7be8321be4d12e20e 2024-11-08T00:24:55,457 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/beb16b12b92b41f7be8321be4d12e20e, entries=150, sequenceid=434, filesize=12.0 K 2024-11-08T00:24:55,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/be27ea9bd5b846f8b2cd682caba7beb6 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/be27ea9bd5b846f8b2cd682caba7beb6 2024-11-08T00:24:55,465 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/be27ea9bd5b846f8b2cd682caba7beb6, entries=150, sequenceid=434, filesize=12.0 K 2024-11-08T00:24:55,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/0808968364b04bedbfa6b574723184de as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/0808968364b04bedbfa6b574723184de 2024-11-08T00:24:55,472 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/0808968364b04bedbfa6b574723184de, entries=150, sequenceid=434, filesize=12.0 K 2024-11-08T00:24:55,474 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 94c140d3911f03ee9ce6997a067b1a83 in 592ms, sequenceid=434, compaction requested=false 2024-11-08T00:24:55,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:24:55,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:55,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-08T00:24:55,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-08T00:24:55,478 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-08T00:24:55,479 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6040 sec 2024-11-08T00:24:55,480 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 2.6110 sec 2024-11-08T00:24:55,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 94c140d3911f03ee9ce6997a067b1a83 2024-11-08T00:24:55,488 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:24:55,488 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94c140d3911f03ee9ce6997a067b1a83 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-08T00:24:55,494 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/5684bfe09bcb412a9ff529bdd2ce2ee7 is 50, key is test_row_0/A:col10/1731025495152/Put/seqid=0 2024-11-08T00:24:55,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742147_1323 (size=12301) 2024-11-08T00:24:55,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:55,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025555525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:55,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:55,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:55,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:55,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:55,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025555528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:55,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:55,532 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:55,532 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:55,532 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:55,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025555528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:55,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:55,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025555530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:55,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:55,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:55,533 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:55,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:55,533 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:55,533 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:55,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:55,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025555637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:55,641 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:55,641 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:55,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:55,641 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:55,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025555637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:55,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:55,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:55,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:55,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:55,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025555638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:55,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:55,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025555638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:55,642 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:55,642 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:55,642 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:55,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:55,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:55,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:55,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:55,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025555848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:55,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:55,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:55,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:55,849 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:55,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025555848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:55,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:55,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:55,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:55,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:55,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025555848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:55,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:55,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:55,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:55,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:55,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025555857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:55,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:55,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:55,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:55,916 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/5684bfe09bcb412a9ff529bdd2ce2ee7 2024-11-08T00:24:55,958 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/ef0a39c55d1d42e4942e2570af10f8b8 is 50, key is test_row_0/B:col10/1731025495152/Put/seqid=0 2024-11-08T00:24:55,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742148_1324 (size=12301) 2024-11-08T00:24:56,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:56,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025556157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:56,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:56,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:56,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:56,160 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:56,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025556158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:56,161 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:56,161 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:56,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:56,161 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:56,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025556158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:56,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:56,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:56,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:56,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:56,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025556167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:56,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:56,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:56,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:56,393 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/ef0a39c55d1d42e4942e2570af10f8b8 2024-11-08T00:24:56,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/2d61f78fe9ed46469851a0ea4c30ab56 is 50, key is test_row_0/C:col10/1731025495152/Put/seqid=0 2024-11-08T00:24:56,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742149_1325 (size=12301) 2024-11-08T00:24:56,404 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/2d61f78fe9ed46469851a0ea4c30ab56 2024-11-08T00:24:56,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/5684bfe09bcb412a9ff529bdd2ce2ee7 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/5684bfe09bcb412a9ff529bdd2ce2ee7 2024-11-08T00:24:56,420 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/5684bfe09bcb412a9ff529bdd2ce2ee7, entries=150, sequenceid=452, filesize=12.0 K 2024-11-08T00:24:56,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/ef0a39c55d1d42e4942e2570af10f8b8 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/ef0a39c55d1d42e4942e2570af10f8b8 2024-11-08T00:24:56,425 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/ef0a39c55d1d42e4942e2570af10f8b8, entries=150, sequenceid=452, filesize=12.0 K 2024-11-08T00:24:56,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/2d61f78fe9ed46469851a0ea4c30ab56 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/2d61f78fe9ed46469851a0ea4c30ab56 2024-11-08T00:24:56,432 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/2d61f78fe9ed46469851a0ea4c30ab56, entries=150, sequenceid=452, filesize=12.0 K 2024-11-08T00:24:56,433 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 94c140d3911f03ee9ce6997a067b1a83 in 945ms, sequenceid=452, compaction requested=true 2024-11-08T00:24:56,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:24:56,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c140d3911f03ee9ce6997a067b1a83:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T00:24:56,433 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:24:56,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:56,433 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:24:56,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c140d3911f03ee9ce6997a067b1a83:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T00:24:56,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:56,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c140d3911f03ee9ce6997a067b1a83:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:24:56,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:24:56,436 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:24:56,436 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:24:56,436 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): 94c140d3911f03ee9ce6997a067b1a83/A is initiating minor compaction (all files) 2024-11-08T00:24:56,436 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): 94c140d3911f03ee9ce6997a067b1a83/B is initiating minor compaction (all files) 2024-11-08T00:24:56,436 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c140d3911f03ee9ce6997a067b1a83/B in TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:56,436 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c140d3911f03ee9ce6997a067b1a83/A in TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:56,436 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/60111580c7f545e5a5c29d464e7e7324, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/beb16b12b92b41f7be8321be4d12e20e, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/5684bfe09bcb412a9ff529bdd2ce2ee7] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp, totalSize=37.0 K 2024-11-08T00:24:56,436 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/b45e45c889ca45c1bf14521a41adbe43, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/be27ea9bd5b846f8b2cd682caba7beb6, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/ef0a39c55d1d42e4942e2570af10f8b8] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp, totalSize=37.0 K 2024-11-08T00:24:56,436 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting b45e45c889ca45c1bf14521a41adbe43, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1731025492811 2024-11-08T00:24:56,436 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60111580c7f545e5a5c29d464e7e7324, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1731025492811 2024-11-08T00:24:56,437 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting beb16b12b92b41f7be8321be4d12e20e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1731025493984 2024-11-08T00:24:56,437 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting be27ea9bd5b846f8b2cd682caba7beb6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1731025493984 2024-11-08T00:24:56,438 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5684bfe09bcb412a9ff529bdd2ce2ee7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1731025495151 2024-11-08T00:24:56,438 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting ef0a39c55d1d42e4942e2570af10f8b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1731025495151 2024-11-08T00:24:56,450 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c140d3911f03ee9ce6997a067b1a83#A#compaction#278 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:56,450 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/1e2b041e45874d70bbf66c0c0c0cbb1b is 50, key is test_row_0/A:col10/1731025495152/Put/seqid=0 2024-11-08T00:24:56,458 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c140d3911f03ee9ce6997a067b1a83#B#compaction#279 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:56,459 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/298ae7cb318f4608a729c3185d167b3b is 50, key is test_row_0/B:col10/1731025495152/Put/seqid=0 2024-11-08T00:24:56,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742150_1326 (size=13357) 2024-11-08T00:24:56,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742151_1327 (size=13357) 2024-11-08T00:24:56,475 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/1e2b041e45874d70bbf66c0c0c0cbb1b as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/1e2b041e45874d70bbf66c0c0c0cbb1b 2024-11-08T00:24:56,483 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/298ae7cb318f4608a729c3185d167b3b as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/298ae7cb318f4608a729c3185d167b3b 2024-11-08T00:24:56,494 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94c140d3911f03ee9ce6997a067b1a83/B of 94c140d3911f03ee9ce6997a067b1a83 into 298ae7cb318f4608a729c3185d167b3b(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:56,494 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94c140d3911f03ee9ce6997a067b1a83/A of 94c140d3911f03ee9ce6997a067b1a83 into 1e2b041e45874d70bbf66c0c0c0cbb1b(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:56,494 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:24:56,494 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:24:56,494 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., storeName=94c140d3911f03ee9ce6997a067b1a83/A, priority=13, startTime=1731025496433; duration=0sec 2024-11-08T00:24:56,494 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., storeName=94c140d3911f03ee9ce6997a067b1a83/B, priority=13, startTime=1731025496433; duration=0sec 2024-11-08T00:24:56,494 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:24:56,494 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c140d3911f03ee9ce6997a067b1a83:A 2024-11-08T00:24:56,494 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:56,494 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:24:56,494 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c140d3911f03ee9ce6997a067b1a83:B 2024-11-08T00:24:56,496 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:24:56,496 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): 94c140d3911f03ee9ce6997a067b1a83/C is initiating minor compaction (all files) 2024-11-08T00:24:56,496 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c140d3911f03ee9ce6997a067b1a83/C in TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:56,496 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/b3bfd60cbc764057ac2dfce140533c21, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/0808968364b04bedbfa6b574723184de, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/2d61f78fe9ed46469851a0ea4c30ab56] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp, totalSize=36.9 K 2024-11-08T00:24:56,496 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3bfd60cbc764057ac2dfce140533c21, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1731025492811 2024-11-08T00:24:56,497 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0808968364b04bedbfa6b574723184de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1731025493984 2024-11-08T00:24:56,497 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d61f78fe9ed46469851a0ea4c30ab56, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1731025495151 2024-11-08T00:24:56,504 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c140d3911f03ee9ce6997a067b1a83#C#compaction#280 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:56,505 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/23673bab20584c379061f6c07f0dcb30 is 50, key is test_row_0/C:col10/1731025495152/Put/seqid=0 2024-11-08T00:24:56,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742152_1328 (size=13323) 2024-11-08T00:24:56,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 94c140d3911f03ee9ce6997a067b1a83 2024-11-08T00:24:56,669 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:24:56,669 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94c140d3911f03ee9ce6997a067b1a83 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-08T00:24:56,676 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/4675ed2c003947a89a9f847cc6b9b456 is 50, key is test_row_0/A:col10/1731025495525/Put/seqid=0 2024-11-08T00:24:56,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742153_1329 (size=17181) 2024-11-08T00:24:56,686 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/4675ed2c003947a89a9f847cc6b9b456 2024-11-08T00:24:56,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:56,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025556682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:56,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:56,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:56,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:56,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:56,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025556685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:56,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:56,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:56,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:56,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:56,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025556685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:56,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:56,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025556688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:56,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:56,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:56,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:56,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:56,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:56,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:56,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/8a70e76d01934e628395eeb165ea7fde is 50, key is test_row_0/B:col10/1731025495525/Put/seqid=0 2024-11-08T00:24:56,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742154_1330 (size=12301) 2024-11-08T00:24:56,744 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/8a70e76d01934e628395eeb165ea7fde 2024-11-08T00:24:56,767 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/1db6c23ef4f84660831424741515a5a0 is 50, key is test_row_0/C:col10/1731025495525/Put/seqid=0 2024-11-08T00:24:56,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742155_1331 (size=12301) 2024-11-08T00:24:56,790 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/1db6c23ef4f84660831424741515a5a0 2024-11-08T00:24:56,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/4675ed2c003947a89a9f847cc6b9b456 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/4675ed2c003947a89a9f847cc6b9b456 2024-11-08T00:24:56,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:56,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025556797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:56,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:56,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:56,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:56,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:56,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025556798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:56,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:56,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025556798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:56,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:56,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:56,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:56,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:56,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025556798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:56,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:56,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:56,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:56,803 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/4675ed2c003947a89a9f847cc6b9b456, entries=250, sequenceid=478, filesize=16.8 K 2024-11-08T00:24:56,803 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:56,803 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:56,803 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:56,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/8a70e76d01934e628395eeb165ea7fde as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/8a70e76d01934e628395eeb165ea7fde 2024-11-08T00:24:56,812 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/8a70e76d01934e628395eeb165ea7fde, entries=150, sequenceid=478, filesize=12.0 K 2024-11-08T00:24:56,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/1db6c23ef4f84660831424741515a5a0 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/1db6c23ef4f84660831424741515a5a0 2024-11-08T00:24:56,822 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/1db6c23ef4f84660831424741515a5a0, entries=150, sequenceid=478, filesize=12.0 K 2024-11-08T00:24:56,823 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 94c140d3911f03ee9ce6997a067b1a83 in 154ms, sequenceid=478, compaction requested=false 2024-11-08T00:24:56,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:24:56,940 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/23673bab20584c379061f6c07f0dcb30 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/23673bab20584c379061f6c07f0dcb30 2024-11-08T00:24:56,945 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94c140d3911f03ee9ce6997a067b1a83/C of 94c140d3911f03ee9ce6997a067b1a83 into 23673bab20584c379061f6c07f0dcb30(size=13.0 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:56,945 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:24:56,945 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., storeName=94c140d3911f03ee9ce6997a067b1a83/C, priority=13, startTime=1731025496434; duration=0sec 2024-11-08T00:24:56,946 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:56,946 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c140d3911f03ee9ce6997a067b1a83:C 2024-11-08T00:24:57,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 94c140d3911f03ee9ce6997a067b1a83 2024-11-08T00:24:57,009 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:24:57,009 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94c140d3911f03ee9ce6997a067b1a83 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-08T00:24:57,015 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/38d3d32440cb427eb5ac6ff587002b47 is 50, key is test_row_0/A:col10/1731025496680/Put/seqid=0 2024-11-08T00:24:57,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742156_1332 (size=12301) 2024-11-08T00:24:57,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-08T00:24:57,037 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:24:57,038 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-08T00:24:57,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-08T00:24:57,040 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T00:24:57,041 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T00:24:57,041 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T00:24:57,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-08T00:24:57,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:57,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025557042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:57,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:57,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:57,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:57,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:57,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025557043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:57,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:57,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:57,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025557044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:57,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:57,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:57,049 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:57,049 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:57,049 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:57,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:57,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025557046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:57,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:57,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:57,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:57,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-08T00:24:57,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:57,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025557158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:57,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:57,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:57,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025557158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:57,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:57,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:57,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:57,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:57,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:57,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:57,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025557158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:57,162 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:57,162 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:57,162 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:57,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:57,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025557158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:57,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:57,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:57,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:57,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-08T00:24:57,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:57,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. as already flushing 2024-11-08T00:24:57,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:57,194 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:57,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:57,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:57,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-08T00:24:57,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:57,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. as already flushing 2024-11-08T00:24:57,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:57,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:57,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:57,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:57,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-08T00:24:57,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:57,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025557367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:57,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:57,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025557367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:57,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:57,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025557368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:57,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:57,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:57,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:57,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:57,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025557368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:57,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:57,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:57,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:57,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:57,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:57,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:57,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:57,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:57,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:57,425 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/38d3d32440cb427eb5ac6ff587002b47 2024-11-08T00:24:57,432 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/1710ded12afb4a9c8fb9bf7f85e081f6 is 50, key is test_row_0/B:col10/1731025496680/Put/seqid=0 2024-11-08T00:24:57,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742157_1333 (size=12301) 2024-11-08T00:24:57,451 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/1710ded12afb4a9c8fb9bf7f85e081f6 2024-11-08T00:24:57,463 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/2c1b2deaf60749089b1a2a44ed3172ea is 50, key is test_row_0/C:col10/1731025496680/Put/seqid=0 2024-11-08T00:24:57,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742158_1334 (size=12301) 2024-11-08T00:24:57,492 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/2c1b2deaf60749089b1a2a44ed3172ea 2024-11-08T00:24:57,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/38d3d32440cb427eb5ac6ff587002b47 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/38d3d32440cb427eb5ac6ff587002b47 2024-11-08T00:24:57,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-08T00:24:57,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:57,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. as already flushing 2024-11-08T00:24:57,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:57,499 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:57,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:57,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:57,506 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/38d3d32440cb427eb5ac6ff587002b47, entries=150, sequenceid=493, filesize=12.0 K 2024-11-08T00:24:57,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/1710ded12afb4a9c8fb9bf7f85e081f6 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/1710ded12afb4a9c8fb9bf7f85e081f6 2024-11-08T00:24:57,517 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/1710ded12afb4a9c8fb9bf7f85e081f6, entries=150, sequenceid=493, filesize=12.0 K 2024-11-08T00:24:57,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/2c1b2deaf60749089b1a2a44ed3172ea as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/2c1b2deaf60749089b1a2a44ed3172ea 2024-11-08T00:24:57,526 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/2c1b2deaf60749089b1a2a44ed3172ea, entries=150, sequenceid=493, filesize=12.0 K 2024-11-08T00:24:57,527 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 94c140d3911f03ee9ce6997a067b1a83 in 518ms, sequenceid=493, compaction requested=true 2024-11-08T00:24:57,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:24:57,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c140d3911f03ee9ce6997a067b1a83:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T00:24:57,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:57,527 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:24:57,530 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:24:57,530 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42839 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:24:57,530 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): 94c140d3911f03ee9ce6997a067b1a83/A is initiating minor compaction (all files) 2024-11-08T00:24:57,530 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c140d3911f03ee9ce6997a067b1a83/A in TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:57,530 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/1e2b041e45874d70bbf66c0c0c0cbb1b, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/4675ed2c003947a89a9f847cc6b9b456, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/38d3d32440cb427eb5ac6ff587002b47] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp, totalSize=41.8 K 2024-11-08T00:24:57,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c140d3911f03ee9ce6997a067b1a83:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T00:24:57,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:57,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c140d3911f03ee9ce6997a067b1a83:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:24:57,531 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e2b041e45874d70bbf66c0c0c0cbb1b, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1731025495151 2024-11-08T00:24:57,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:24:57,531 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:24:57,531 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): 94c140d3911f03ee9ce6997a067b1a83/B is initiating minor compaction (all files) 2024-11-08T00:24:57,532 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4675ed2c003947a89a9f847cc6b9b456, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=478, earliestPutTs=1731025495525 2024-11-08T00:24:57,532 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c140d3911f03ee9ce6997a067b1a83/B in TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:57,532 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/298ae7cb318f4608a729c3185d167b3b, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/8a70e76d01934e628395eeb165ea7fde, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/1710ded12afb4a9c8fb9bf7f85e081f6] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp, totalSize=37.1 K 2024-11-08T00:24:57,532 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38d3d32440cb427eb5ac6ff587002b47, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1731025496680 2024-11-08T00:24:57,532 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 298ae7cb318f4608a729c3185d167b3b, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1731025495151 2024-11-08T00:24:57,532 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a70e76d01934e628395eeb165ea7fde, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=478, earliestPutTs=1731025495525 2024-11-08T00:24:57,533 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 1710ded12afb4a9c8fb9bf7f85e081f6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1731025496680 2024-11-08T00:24:57,552 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c140d3911f03ee9ce6997a067b1a83#A#compaction#287 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:57,552 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/bfaed82349a54adc9bb250a2b10584fb is 50, key is test_row_0/A:col10/1731025496680/Put/seqid=0 2024-11-08T00:24:57,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742159_1335 (size=13459) 2024-11-08T00:24:57,569 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c140d3911f03ee9ce6997a067b1a83#B#compaction#288 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:57,570 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/855b06d275a74b2f9c0d7fa283e1c6d4 is 50, key is test_row_0/B:col10/1731025496680/Put/seqid=0 2024-11-08T00:24:57,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742160_1336 (size=13459) 2024-11-08T00:24:57,585 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/bfaed82349a54adc9bb250a2b10584fb as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/bfaed82349a54adc9bb250a2b10584fb 2024-11-08T00:24:57,591 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94c140d3911f03ee9ce6997a067b1a83/A of 94c140d3911f03ee9ce6997a067b1a83 into bfaed82349a54adc9bb250a2b10584fb(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:57,591 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:24:57,591 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., storeName=94c140d3911f03ee9ce6997a067b1a83/A, priority=13, startTime=1731025497527; duration=0sec 2024-11-08T00:24:57,592 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:24:57,592 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c140d3911f03ee9ce6997a067b1a83:A 2024-11-08T00:24:57,592 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:24:57,593 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:24:57,593 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): 94c140d3911f03ee9ce6997a067b1a83/C is initiating minor compaction (all files) 2024-11-08T00:24:57,594 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c140d3911f03ee9ce6997a067b1a83/C in TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:57,594 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/23673bab20584c379061f6c07f0dcb30, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/1db6c23ef4f84660831424741515a5a0, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/2c1b2deaf60749089b1a2a44ed3172ea] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp, totalSize=37.0 K 2024-11-08T00:24:57,594 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23673bab20584c379061f6c07f0dcb30, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1731025495151 2024-11-08T00:24:57,594 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1db6c23ef4f84660831424741515a5a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=478, earliestPutTs=1731025495525 2024-11-08T00:24:57,595 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c1b2deaf60749089b1a2a44ed3172ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1731025496680 2024-11-08T00:24:57,606 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c140d3911f03ee9ce6997a067b1a83#C#compaction#289 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:57,608 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/1c8fc20bf7f34952a7cd72c00861e9ec is 50, key is test_row_0/C:col10/1731025496680/Put/seqid=0 2024-11-08T00:24:57,652 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-08T00:24:57,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:57,653 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 94c140d3911f03ee9ce6997a067b1a83 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-08T00:24:57,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742161_1337 (size=13425) 2024-11-08T00:24:57,665 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/1c8fc20bf7f34952a7cd72c00861e9ec as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/1c8fc20bf7f34952a7cd72c00861e9ec 2024-11-08T00:24:57,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-08T00:24:57,673 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94c140d3911f03ee9ce6997a067b1a83/C of 94c140d3911f03ee9ce6997a067b1a83 into 1c8fc20bf7f34952a7cd72c00861e9ec(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:57,673 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:24:57,673 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., storeName=94c140d3911f03ee9ce6997a067b1a83/C, priority=13, startTime=1731025497531; duration=0sec 2024-11-08T00:24:57,673 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:57,674 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c140d3911f03ee9ce6997a067b1a83:C 2024-11-08T00:24:57,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/063a45b4bdfe42d5af282360da9ea617 is 50, key is test_row_0/A:col10/1731025497041/Put/seqid=0 2024-11-08T00:24:57,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 94c140d3911f03ee9ce6997a067b1a83 2024-11-08T00:24:57,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. as already flushing 2024-11-08T00:24:57,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742162_1338 (size=12301) 2024-11-08T00:24:57,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:57,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025557696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:57,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:57,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025557696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:57,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:57,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:57,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:57,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:57,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45704 deadline: 1731025557697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:57,701 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:57,701 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:57,701 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:57,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:57,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:57,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025557697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:57,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025557697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:57,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:57,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:57,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:57,702 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 3302f0f507bd,40479,1731025415856 for 'test_row_2' in 94c140d3911f03ee9ce6997a067b1a83 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T00:24:57,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:57,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:57,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:57,703 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:57,703 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:57,703 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:57,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:57,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025557807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:57,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:57,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:57,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025557807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:57,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:57,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:57,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:57,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025557807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:57,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:57,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:57,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:57,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:57,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:57,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:57,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:57,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025557808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:57,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:57,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:57,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:58,008 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/855b06d275a74b2f9c0d7fa283e1c6d4 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/855b06d275a74b2f9c0d7fa283e1c6d4 2024-11-08T00:24:58,018 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94c140d3911f03ee9ce6997a067b1a83/B of 94c140d3911f03ee9ce6997a067b1a83 into 855b06d275a74b2f9c0d7fa283e1c6d4(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:58,018 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:24:58,018 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., storeName=94c140d3911f03ee9ce6997a067b1a83/B, priority=13, startTime=1731025497530; duration=0sec 2024-11-08T00:24:58,018 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:58,018 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c140d3911f03ee9ce6997a067b1a83:B 2024-11-08T00:24:58,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:58,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025558018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:58,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:58,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:58,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:58,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:58,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025558018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:58,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:58,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:58,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:58,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:58,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025558019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:58,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:58,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025558019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:58,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:58,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:58,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:58,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:58,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:58,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:58,097 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=516 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/063a45b4bdfe42d5af282360da9ea617 2024-11-08T00:24:58,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/556e07fb27d74523ab33fd82e973268a is 50, key is test_row_0/B:col10/1731025497041/Put/seqid=0 2024-11-08T00:24:58,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-08T00:24:58,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742163_1339 (size=12301) 2024-11-08T00:24:58,193 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=516 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/556e07fb27d74523ab33fd82e973268a 2024-11-08T00:24:58,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/f8233d534b8f4f208928dcc9ddc0d827 is 50, key is test_row_0/C:col10/1731025497041/Put/seqid=0 2024-11-08T00:24:58,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742164_1340 (size=12301) 2024-11-08T00:24:58,267 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=516 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/f8233d534b8f4f208928dcc9ddc0d827 2024-11-08T00:24:58,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/063a45b4bdfe42d5af282360da9ea617 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/063a45b4bdfe42d5af282360da9ea617 2024-11-08T00:24:58,282 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/063a45b4bdfe42d5af282360da9ea617, entries=150, sequenceid=516, filesize=12.0 K 2024-11-08T00:24:58,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/556e07fb27d74523ab33fd82e973268a as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/556e07fb27d74523ab33fd82e973268a 2024-11-08T00:24:58,292 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/556e07fb27d74523ab33fd82e973268a, entries=150, sequenceid=516, filesize=12.0 K 2024-11-08T00:24:58,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/f8233d534b8f4f208928dcc9ddc0d827 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/f8233d534b8f4f208928dcc9ddc0d827 2024-11-08T00:24:58,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:58,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025558368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:58,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:58,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:58,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:58,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:58,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025558368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:58,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:58,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:58,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:58,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:58,372 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/f8233d534b8f4f208928dcc9ddc0d827, entries=150, sequenceid=516, filesize=12.0 K 2024-11-08T00:24:58,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025558368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:58,372 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:58,372 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:58,372 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:58,375 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for 94c140d3911f03ee9ce6997a067b1a83 in 722ms, sequenceid=516, compaction requested=false 2024-11-08T00:24:58,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:24:58,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:58,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-08T00:24:58,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-08T00:24:58,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 94c140d3911f03ee9ce6997a067b1a83 2024-11-08T00:24:58,378 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:24:58,378 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94c140d3911f03ee9ce6997a067b1a83 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-08T00:24:58,384 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-08T00:24:58,384 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3370 sec 2024-11-08T00:24:58,389 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.3490 sec 2024-11-08T00:24:58,399 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/e00aaf4124114beabf4f1ae17367dbbe is 50, key is test_row_0/A:col10/1731025497696/Put/seqid=0 2024-11-08T00:24:58,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742165_1341 (size=14741) 2024-11-08T00:24:58,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:58,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025558553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:58,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:58,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:58,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:58,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:58,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025558667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:58,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:58,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:58,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:58,843 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=534 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/e00aaf4124114beabf4f1ae17367dbbe 2024-11-08T00:24:58,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/424325aa8df4435d94750b10917ef85d is 50, key is test_row_0/B:col10/1731025497696/Put/seqid=0 2024-11-08T00:24:58,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:58,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025558879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:58,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:58,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:58,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:58,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:58,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025558879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:58,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:58,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:58,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:58,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:58,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025558879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:58,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:58,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:58,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:58,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:58,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025558888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:58,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:58,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:58,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:58,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742166_1342 (size=12301) 2024-11-08T00:24:58,942 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=534 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/424325aa8df4435d94750b10917ef85d 2024-11-08T00:24:58,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/f531e471711942ef82eccd33b081e1c0 is 50, key is test_row_0/C:col10/1731025497696/Put/seqid=0 2024-11-08T00:24:59,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742167_1343 (size=12301) 2024-11-08T00:24:59,047 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=534 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/f531e471711942ef82eccd33b081e1c0 2024-11-08T00:24:59,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/e00aaf4124114beabf4f1ae17367dbbe as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/e00aaf4124114beabf4f1ae17367dbbe 2024-11-08T00:24:59,064 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/e00aaf4124114beabf4f1ae17367dbbe, entries=200, sequenceid=534, filesize=14.4 K 2024-11-08T00:24:59,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/424325aa8df4435d94750b10917ef85d as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/424325aa8df4435d94750b10917ef85d 2024-11-08T00:24:59,078 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/424325aa8df4435d94750b10917ef85d, entries=150, sequenceid=534, filesize=12.0 K 2024-11-08T00:24:59,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/f531e471711942ef82eccd33b081e1c0 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/f531e471711942ef82eccd33b081e1c0 2024-11-08T00:24:59,092 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/f531e471711942ef82eccd33b081e1c0, entries=150, sequenceid=534, filesize=12.0 K 2024-11-08T00:24:59,093 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 94c140d3911f03ee9ce6997a067b1a83 in 715ms, sequenceid=534, compaction requested=true 2024-11-08T00:24:59,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:24:59,094 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:24:59,095 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40501 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:24:59,096 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): 94c140d3911f03ee9ce6997a067b1a83/A is initiating minor compaction (all files) 2024-11-08T00:24:59,096 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c140d3911f03ee9ce6997a067b1a83/A in TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:59,096 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/bfaed82349a54adc9bb250a2b10584fb, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/063a45b4bdfe42d5af282360da9ea617, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/e00aaf4124114beabf4f1ae17367dbbe] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp, totalSize=39.6 K 2024-11-08T00:24:59,097 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting bfaed82349a54adc9bb250a2b10584fb, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1731025496680 2024-11-08T00:24:59,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c140d3911f03ee9ce6997a067b1a83:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T00:24:59,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:59,097 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 063a45b4bdfe42d5af282360da9ea617, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=516, earliestPutTs=1731025497039 2024-11-08T00:24:59,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c140d3911f03ee9ce6997a067b1a83:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T00:24:59,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:59,097 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:24:59,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c140d3911f03ee9ce6997a067b1a83:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:24:59,098 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting e00aaf4124114beabf4f1ae17367dbbe, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=534, earliestPutTs=1731025497680 2024-11-08T00:24:59,099 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38061 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:24:59,100 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): 94c140d3911f03ee9ce6997a067b1a83/B is initiating minor compaction (all files) 2024-11-08T00:24:59,100 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c140d3911f03ee9ce6997a067b1a83/B in TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:59,100 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/855b06d275a74b2f9c0d7fa283e1c6d4, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/556e07fb27d74523ab33fd82e973268a, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/424325aa8df4435d94750b10917ef85d] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp, totalSize=37.2 K 2024-11-08T00:24:59,101 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 855b06d275a74b2f9c0d7fa283e1c6d4, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1731025496680 2024-11-08T00:24:59,102 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 556e07fb27d74523ab33fd82e973268a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=516, earliestPutTs=1731025497039 2024-11-08T00:24:59,103 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 424325aa8df4435d94750b10917ef85d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=534, earliestPutTs=1731025497680 2024-11-08T00:24:59,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:24:59,125 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c140d3911f03ee9ce6997a067b1a83#A#compaction#296 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:59,125 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/240dcb2e56524f0f86701e331e3c4753 is 50, key is test_row_0/A:col10/1731025497696/Put/seqid=0 2024-11-08T00:24:59,128 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c140d3911f03ee9ce6997a067b1a83#B#compaction#297 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:59,129 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/213cfd4dddfa4b35aac514102cda043a is 50, key is test_row_0/B:col10/1731025497696/Put/seqid=0 2024-11-08T00:24:59,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-08T00:24:59,188 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:24:59,188 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-08T00:24:59,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-08T00:24:59,193 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T00:24:59,194 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T00:24:59,194 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T00:24:59,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-08T00:24:59,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 94c140d3911f03ee9ce6997a067b1a83 2024-11-08T00:24:59,204 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:24:59,204 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94c140d3911f03ee9ce6997a067b1a83 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-08T00:24:59,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742168_1344 (size=13561) 2024-11-08T00:24:59,233 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/213cfd4dddfa4b35aac514102cda043a as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/213cfd4dddfa4b35aac514102cda043a 2024-11-08T00:24:59,240 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94c140d3911f03ee9ce6997a067b1a83/B of 94c140d3911f03ee9ce6997a067b1a83 into 213cfd4dddfa4b35aac514102cda043a(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:59,241 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:24:59,241 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., storeName=94c140d3911f03ee9ce6997a067b1a83/B, priority=13, startTime=1731025499097; duration=0sec 2024-11-08T00:24:59,241 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:24:59,248 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c140d3911f03ee9ce6997a067b1a83:B 2024-11-08T00:24:59,249 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:24:59,250 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:24:59,250 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): 94c140d3911f03ee9ce6997a067b1a83/C is initiating minor compaction (all files) 2024-11-08T00:24:59,250 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c140d3911f03ee9ce6997a067b1a83/C in TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:59,250 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/1c8fc20bf7f34952a7cd72c00861e9ec, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/f8233d534b8f4f208928dcc9ddc0d827, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/f531e471711942ef82eccd33b081e1c0] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp, totalSize=37.1 K 2024-11-08T00:24:59,251 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c8fc20bf7f34952a7cd72c00861e9ec, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1731025496680 2024-11-08T00:24:59,251 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting f8233d534b8f4f208928dcc9ddc0d827, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=516, earliestPutTs=1731025497039 2024-11-08T00:24:59,252 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting f531e471711942ef82eccd33b081e1c0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=534, earliestPutTs=1731025497680 2024-11-08T00:24:59,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742169_1345 (size=13561) 2024-11-08T00:24:59,273 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/240dcb2e56524f0f86701e331e3c4753 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/240dcb2e56524f0f86701e331e3c4753 2024-11-08T00:24:59,279 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/80e65fd512fd45d5aacdd9c3bec4c2a6 is 50, key is test_row_0/A:col10/1731025498538/Put/seqid=0 2024-11-08T00:24:59,283 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94c140d3911f03ee9ce6997a067b1a83/A of 94c140d3911f03ee9ce6997a067b1a83 into 240dcb2e56524f0f86701e331e3c4753(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:59,283 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:24:59,283 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., storeName=94c140d3911f03ee9ce6997a067b1a83/A, priority=13, startTime=1731025499094; duration=0sec 2024-11-08T00:24:59,283 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:59,283 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c140d3911f03ee9ce6997a067b1a83:A 2024-11-08T00:24:59,300 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c140d3911f03ee9ce6997a067b1a83#C#compaction#299 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:24:59,301 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/f8eef3d328c74293a17d18312e1e1942 is 50, key is test_row_0/C:col10/1731025497696/Put/seqid=0 2024-11-08T00:24:59,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-08T00:24:59,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742171_1347 (size=13527) 2024-11-08T00:24:59,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742170_1346 (size=14741) 2024-11-08T00:24:59,324 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/f8eef3d328c74293a17d18312e1e1942 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/f8eef3d328c74293a17d18312e1e1942 2024-11-08T00:24:59,332 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94c140d3911f03ee9ce6997a067b1a83/C of 94c140d3911f03ee9ce6997a067b1a83 into f8eef3d328c74293a17d18312e1e1942(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:24:59,332 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:24:59,332 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., storeName=94c140d3911f03ee9ce6997a067b1a83/C, priority=13, startTime=1731025499097; duration=0sec 2024-11-08T00:24:59,332 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:24:59,332 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c140d3911f03ee9ce6997a067b1a83:C 2024-11-08T00:24:59,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-08T00:24:59,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:59,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. as already flushing 2024-11-08T00:24:59,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:59,354 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:59,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:59,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:59,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:59,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025559372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:59,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:59,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:59,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:59,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:59,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025559487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:59,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:59,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:59,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:59,508 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-08T00:24:59,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:59,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. as already flushing 2024-11-08T00:24:59,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:59,509 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:59,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:59,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:59,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-08T00:24:59,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-08T00:24:59,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:59,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. as already flushing 2024-11-08T00:24:59,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:59,666 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:59,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:59,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:59,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:59,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025559707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:59,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:59,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:59,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:59,718 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=555 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/80e65fd512fd45d5aacdd9c3bec4c2a6 2024-11-08T00:24:59,742 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/6634a4337da14bc490cdff5f1c465015 is 50, key is test_row_0/B:col10/1731025498538/Put/seqid=0 2024-11-08T00:24:59,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742172_1348 (size=12301) 2024-11-08T00:24:59,782 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=555 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/6634a4337da14bc490cdff5f1c465015 2024-11-08T00:24:59,818 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/498c204495f7434db91f7ea78b009ce0 is 50, key is test_row_0/C:col10/1731025498538/Put/seqid=0 2024-11-08T00:24:59,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-08T00:24:59,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:59,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. as already flushing 2024-11-08T00:24:59,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:59,820 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:59,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:59,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:59,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-08T00:24:59,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742173_1349 (size=12301) 2024-11-08T00:24:59,855 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=555 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/498c204495f7434db91f7ea78b009ce0 2024-11-08T00:24:59,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/80e65fd512fd45d5aacdd9c3bec4c2a6 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/80e65fd512fd45d5aacdd9c3bec4c2a6 2024-11-08T00:24:59,865 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/80e65fd512fd45d5aacdd9c3bec4c2a6, entries=200, sequenceid=555, filesize=14.4 K 2024-11-08T00:24:59,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/6634a4337da14bc490cdff5f1c465015 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/6634a4337da14bc490cdff5f1c465015 2024-11-08T00:24:59,876 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/6634a4337da14bc490cdff5f1c465015, entries=150, sequenceid=555, filesize=12.0 K 2024-11-08T00:24:59,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/498c204495f7434db91f7ea78b009ce0 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/498c204495f7434db91f7ea78b009ce0 2024-11-08T00:24:59,888 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/498c204495f7434db91f7ea78b009ce0, entries=150, sequenceid=555, filesize=12.0 K 2024-11-08T00:24:59,896 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 94c140d3911f03ee9ce6997a067b1a83 in 692ms, sequenceid=555, compaction requested=false 2024-11-08T00:24:59,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:24:59,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 94c140d3911f03ee9ce6997a067b1a83 2024-11-08T00:24:59,902 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:24:59,902 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94c140d3911f03ee9ce6997a067b1a83 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-08T00:24:59,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/4e9366be52134e40bdd0a4f0ba3c3fdf is 50, key is test_row_0/A:col10/1731025499896/Put/seqid=0 2024-11-08T00:24:59,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742174_1350 (size=19621) 2024-11-08T00:24:59,944 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=576 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/4e9366be52134e40bdd0a4f0ba3c3fdf 2024-11-08T00:24:59,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:59,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025559964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:59,974 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/0fc0325f83b04cdaa1217a94ce583c1f is 50, key is test_row_0/B:col10/1731025499896/Put/seqid=0 2024-11-08T00:24:59,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:59,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:59,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:59,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-08T00:24:59,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:59,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. as already flushing 2024-11-08T00:24:59,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:24:59,981 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:59,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:59,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:24:59,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:59,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025559973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:59,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:59,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:24:59,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:59,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:24:59,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025559977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:24:59,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:59,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:24:59,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:00,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742175_1351 (size=12301) 2024-11-08T00:25:00,014 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=576 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/0fc0325f83b04cdaa1217a94ce583c1f 2024-11-08T00:25:00,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:00,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025560030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:00,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:00,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:00,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:00,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/b8d7fadd8ff54d82b198445659f68f1e is 50, key is test_row_0/C:col10/1731025499896/Put/seqid=0 2024-11-08T00:25:00,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:00,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025560088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:00,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:00,096 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:00,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025560088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:00,096 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:00,096 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:00,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:00,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:00,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:00,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:00,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025560090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:00,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:00,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:00,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:00,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742176_1352 (size=12301) 2024-11-08T00:25:00,117 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=576 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/b8d7fadd8ff54d82b198445659f68f1e 2024-11-08T00:25:00,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-08T00:25:00,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:25:00,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. as already flushing 2024-11-08T00:25:00,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:25:00,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:00,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:00,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:00,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/4e9366be52134e40bdd0a4f0ba3c3fdf as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/4e9366be52134e40bdd0a4f0ba3c3fdf 2024-11-08T00:25:00,155 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/4e9366be52134e40bdd0a4f0ba3c3fdf, entries=300, sequenceid=576, filesize=19.2 K 2024-11-08T00:25:00,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/0fc0325f83b04cdaa1217a94ce583c1f as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/0fc0325f83b04cdaa1217a94ce583c1f 2024-11-08T00:25:00,225 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/0fc0325f83b04cdaa1217a94ce583c1f, entries=150, sequenceid=576, filesize=12.0 K 2024-11-08T00:25:00,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/b8d7fadd8ff54d82b198445659f68f1e as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/b8d7fadd8ff54d82b198445659f68f1e 2024-11-08T00:25:00,230 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/b8d7fadd8ff54d82b198445659f68f1e, entries=150, sequenceid=576, filesize=12.0 K 2024-11-08T00:25:00,236 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=107.34 KB/109920 for 94c140d3911f03ee9ce6997a067b1a83 in 333ms, sequenceid=576, compaction requested=true 2024-11-08T00:25:00,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:25:00,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c140d3911f03ee9ce6997a067b1a83:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T00:25:00,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:25:00,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c140d3911f03ee9ce6997a067b1a83:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T00:25:00,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-08T00:25:00,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c140d3911f03ee9ce6997a067b1a83:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:25:00,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-08T00:25:00,236 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:25:00,236 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:25:00,237 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38129 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:25:00,238 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): 94c140d3911f03ee9ce6997a067b1a83/C is initiating minor compaction (all files) 2024-11-08T00:25:00,238 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c140d3911f03ee9ce6997a067b1a83/C in TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:25:00,238 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/f8eef3d328c74293a17d18312e1e1942, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/498c204495f7434db91f7ea78b009ce0, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/b8d7fadd8ff54d82b198445659f68f1e] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp, totalSize=37.2 K 2024-11-08T00:25:00,238 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 47923 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:25:00,238 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): 94c140d3911f03ee9ce6997a067b1a83/A is initiating minor compaction (all files) 2024-11-08T00:25:00,238 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c140d3911f03ee9ce6997a067b1a83/A in TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:25:00,238 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/240dcb2e56524f0f86701e331e3c4753, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/80e65fd512fd45d5aacdd9c3bec4c2a6, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/4e9366be52134e40bdd0a4f0ba3c3fdf] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp, totalSize=46.8 K 2024-11-08T00:25:00,240 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 240dcb2e56524f0f86701e331e3c4753, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=534, earliestPutTs=1731025497680 2024-11-08T00:25:00,240 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting f8eef3d328c74293a17d18312e1e1942, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=534, earliestPutTs=1731025497680 2024-11-08T00:25:00,241 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 498c204495f7434db91f7ea78b009ce0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=555, earliestPutTs=1731025498538 2024-11-08T00:25:00,242 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80e65fd512fd45d5aacdd9c3bec4c2a6, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=555, earliestPutTs=1731025498509 2024-11-08T00:25:00,243 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e9366be52134e40bdd0a4f0ba3c3fdf, keycount=300, bloomtype=ROW, size=19.2 K, encoding=NONE, compression=NONE, seqNum=576, earliestPutTs=1731025499262 2024-11-08T00:25:00,243 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting b8d7fadd8ff54d82b198445659f68f1e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=576, earliestPutTs=1731025499345 2024-11-08T00:25:00,266 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c140d3911f03ee9ce6997a067b1a83#A#compaction#305 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:25:00,267 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/4682bf4aef774af29acc05a94b80b21d is 50, key is test_row_0/A:col10/1731025499896/Put/seqid=0 2024-11-08T00:25:00,286 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c140d3911f03ee9ce6997a067b1a83#C#compaction#306 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:25:00,287 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/ee81d2d99ddb4603a28e65162d42323b is 50, key is test_row_0/C:col10/1731025499896/Put/seqid=0 2024-11-08T00:25:00,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-08T00:25:00,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:25:00,296 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 94c140d3911f03ee9ce6997a067b1a83 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-08T00:25:00,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. as already flushing 2024-11-08T00:25:00,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 94c140d3911f03ee9ce6997a067b1a83 2024-11-08T00:25:00,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-08T00:25:00,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742177_1353 (size=13663) 2024-11-08T00:25:00,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742178_1354 (size=13629) 2024-11-08T00:25:00,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/1e9f1c3027b34c6fb05d71e064b4b992 is 50, key is test_row_0/A:col10/1731025499939/Put/seqid=0 2024-11-08T00:25:00,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:00,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025560393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:00,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:00,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:00,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:00,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:00,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025560405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:00,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:00,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025560406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:00,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:00,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:00,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:00,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:00,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:00,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:00,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742179_1355 (size=12301) 2024-11-08T00:25:00,429 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=595 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/1e9f1c3027b34c6fb05d71e064b4b992 2024-11-08T00:25:00,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/99e0deaac2724831ab3e5bbb9ffb9e67 is 50, key is test_row_0/B:col10/1731025499939/Put/seqid=0 2024-11-08T00:25:00,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742180_1356 (size=12301) 2024-11-08T00:25:00,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:00,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025560517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:00,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:00,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:00,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:00,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:00,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025560527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:00,534 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:00,534 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:00,534 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:00,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:00,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025560529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:00,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:00,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:00,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:00,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:00,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45734 deadline: 1731025560547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:00,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:00,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:00,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:00,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:00,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025560727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:00,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:00,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:00,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:00,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:00,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025560738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:00,744 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:00,744 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:00,744 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:00,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:00,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025560752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:00,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:00,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:00,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:00,757 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/4682bf4aef774af29acc05a94b80b21d as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/4682bf4aef774af29acc05a94b80b21d 2024-11-08T00:25:00,762 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94c140d3911f03ee9ce6997a067b1a83/A of 94c140d3911f03ee9ce6997a067b1a83 into 4682bf4aef774af29acc05a94b80b21d(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:25:00,762 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:25:00,762 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., storeName=94c140d3911f03ee9ce6997a067b1a83/A, priority=13, startTime=1731025500236; duration=0sec 2024-11-08T00:25:00,762 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:25:00,762 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c140d3911f03ee9ce6997a067b1a83:A 2024-11-08T00:25:00,762 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:25:00,763 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38163 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:25:00,763 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): 94c140d3911f03ee9ce6997a067b1a83/B is initiating minor compaction (all files) 2024-11-08T00:25:00,763 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c140d3911f03ee9ce6997a067b1a83/B in TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:25:00,763 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/213cfd4dddfa4b35aac514102cda043a, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/6634a4337da14bc490cdff5f1c465015, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/0fc0325f83b04cdaa1217a94ce583c1f] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp, totalSize=37.3 K 2024-11-08T00:25:00,765 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 213cfd4dddfa4b35aac514102cda043a, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=534, earliestPutTs=1731025497680 2024-11-08T00:25:00,769 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6634a4337da14bc490cdff5f1c465015, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=555, earliestPutTs=1731025498538 2024-11-08T00:25:00,771 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0fc0325f83b04cdaa1217a94ce583c1f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=576, earliestPutTs=1731025499345 2024-11-08T00:25:00,775 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/ee81d2d99ddb4603a28e65162d42323b as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/ee81d2d99ddb4603a28e65162d42323b 2024-11-08T00:25:00,781 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94c140d3911f03ee9ce6997a067b1a83/C of 94c140d3911f03ee9ce6997a067b1a83 into ee81d2d99ddb4603a28e65162d42323b(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:25:00,781 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:25:00,781 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., storeName=94c140d3911f03ee9ce6997a067b1a83/C, priority=13, startTime=1731025500236; duration=0sec 2024-11-08T00:25:00,781 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:25:00,781 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c140d3911f03ee9ce6997a067b1a83:C 2024-11-08T00:25:00,787 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c140d3911f03ee9ce6997a067b1a83#B#compaction#309 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:25:00,788 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/5bc479e62b274c05bb7af58a1aa6f9b2 is 50, key is test_row_0/B:col10/1731025499896/Put/seqid=0 2024-11-08T00:25:00,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742181_1357 (size=13663) 2024-11-08T00:25:00,847 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/5bc479e62b274c05bb7af58a1aa6f9b2 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/5bc479e62b274c05bb7af58a1aa6f9b2 2024-11-08T00:25:00,857 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94c140d3911f03ee9ce6997a067b1a83/B of 94c140d3911f03ee9ce6997a067b1a83 into 5bc479e62b274c05bb7af58a1aa6f9b2(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:25:00,857 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:25:00,857 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., storeName=94c140d3911f03ee9ce6997a067b1a83/B, priority=13, startTime=1731025500236; duration=0sec 2024-11-08T00:25:00,857 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:25:00,857 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c140d3911f03ee9ce6997a067b1a83:B 2024-11-08T00:25:00,918 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=595 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/99e0deaac2724831ab3e5bbb9ffb9e67 2024-11-08T00:25:00,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/f599f458dfed4a83a548673977854a1e is 50, key is test_row_0/C:col10/1731025499939/Put/seqid=0 2024-11-08T00:25:00,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742182_1358 (size=12301) 2024-11-08T00:25:01,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:01,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731025561039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:01,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:01,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:01,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:01,055 INFO [Thread-1110 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1110. 2024-11-08T00:25:01,055 DEBUG [Thread-1110 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T00:25:01,055 DEBUG [Thread-1110 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:01,055 DEBUG [Thread-1110 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:01,055 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:25:01,057 INFO [Thread-1104 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1104. 2024-11-08T00:25:01,057 DEBUG [Thread-1104 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T00:25:01,057 DEBUG [Thread-1104 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:01,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:01,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45668 deadline: 1731025561057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:01,058 INFO [Thread-1102 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1102. 2024-11-08T00:25:01,058 DEBUG [Thread-1102 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T00:25:01,058 DEBUG [Thread-1102 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:01,059 DEBUG [Thread-1102 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:01,059 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:01,059 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:01,059 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:01,059 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:25:01,060 INFO [Thread-1108 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1108. 2024-11-08T00:25:01,060 DEBUG [Thread-1108 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T00:25:01,060 DEBUG [Thread-1108 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:01,060 DEBUG [Thread-1108 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:01,060 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:25:01,061 DEBUG [Thread-1104 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:01,061 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:25:01,062 INFO [Thread-1106 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1106. 2024-11-08T00:25:01,062 DEBUG [Thread-1106 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T00:25:01,062 DEBUG [Thread-1106 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:01,062 DEBUG [Thread-1106 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:01,062 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:25:01,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:01,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45670 deadline: 1731025561067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:01,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:01,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:01,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:01,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-08T00:25:01,373 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=595 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/f599f458dfed4a83a548673977854a1e 2024-11-08T00:25:01,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/1e9f1c3027b34c6fb05d71e064b4b992 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/1e9f1c3027b34c6fb05d71e064b4b992 2024-11-08T00:25:01,382 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/1e9f1c3027b34c6fb05d71e064b4b992, entries=150, sequenceid=595, filesize=12.0 K 2024-11-08T00:25:01,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/99e0deaac2724831ab3e5bbb9ffb9e67 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/99e0deaac2724831ab3e5bbb9ffb9e67 2024-11-08T00:25:01,392 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/99e0deaac2724831ab3e5bbb9ffb9e67, entries=150, sequenceid=595, filesize=12.0 K 2024-11-08T00:25:01,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/f599f458dfed4a83a548673977854a1e as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/f599f458dfed4a83a548673977854a1e 2024-11-08T00:25:01,411 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/f599f458dfed4a83a548673977854a1e, entries=150, sequenceid=595, filesize=12.0 K 2024-11-08T00:25:01,412 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 94c140d3911f03ee9ce6997a067b1a83 in 1116ms, sequenceid=595, compaction requested=false 2024-11-08T00:25:01,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:25:01,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:25:01,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-08T00:25:01,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-08T00:25:01,416 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-08T00:25:01,416 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2200 sec 2024-11-08T00:25:01,417 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 2.2280 sec 2024-11-08T00:25:01,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 94c140d3911f03ee9ce6997a067b1a83 2024-11-08T00:25:01,548 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:25:01,548 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 94c140d3911f03ee9ce6997a067b1a83 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-08T00:25:01,548 INFO [Thread-1095 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1095. 2024-11-08T00:25:01,548 DEBUG [Thread-1095 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T00:25:01,548 DEBUG [Thread-1095 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:01,549 DEBUG [Thread-1095 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:01,549 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:25:01,557 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/595e8ed355664677a75b4bf4fd66b16e is 50, key is test_row_0/A:col10/1731025500398/Put/seqid=0 2024-11-08T00:25:01,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742183_1359 (size=12301) 2024-11-08T00:25:01,569 INFO [Thread-1099 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1099. 2024-11-08T00:25:01,569 DEBUG [Thread-1099 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T00:25:01,569 DEBUG [Thread-1099 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:01,569 DEBUG [Thread-1099 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:01,569 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:25:01,570 INFO [Thread-1097 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1097. 2024-11-08T00:25:01,570 DEBUG [Thread-1097 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T00:25:01,570 DEBUG [Thread-1097 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:01,570 DEBUG [Thread-1097 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:01,570 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:25:01,579 INFO [Thread-1091 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1091. 2024-11-08T00:25:01,579 DEBUG [Thread-1091 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T00:25:01,579 DEBUG [Thread-1091 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:01,579 DEBUG [Thread-1091 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:01,579 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:25:01,970 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=616 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/595e8ed355664677a75b4bf4fd66b16e 2024-11-08T00:25:01,977 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/363342f9d45a4c79b84609f8fdbdfa47 is 50, key is test_row_0/B:col10/1731025500398/Put/seqid=0 2024-11-08T00:25:01,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742184_1360 (size=12301) 2024-11-08T00:25:02,380 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=616 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/363342f9d45a4c79b84609f8fdbdfa47 2024-11-08T00:25:02,388 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/a9a70570fd034e85baccf77cd60875ba is 50, key is test_row_0/C:col10/1731025500398/Put/seqid=0 2024-11-08T00:25:02,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742185_1361 (size=12301) 2024-11-08T00:25:02,794 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=616 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/a9a70570fd034e85baccf77cd60875ba 2024-11-08T00:25:02,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/595e8ed355664677a75b4bf4fd66b16e as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/595e8ed355664677a75b4bf4fd66b16e 2024-11-08T00:25:02,808 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/595e8ed355664677a75b4bf4fd66b16e, entries=150, sequenceid=616, filesize=12.0 K 2024-11-08T00:25:02,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/363342f9d45a4c79b84609f8fdbdfa47 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/363342f9d45a4c79b84609f8fdbdfa47 2024-11-08T00:25:02,813 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/363342f9d45a4c79b84609f8fdbdfa47, entries=150, sequenceid=616, filesize=12.0 K 2024-11-08T00:25:02,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/a9a70570fd034e85baccf77cd60875ba as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/a9a70570fd034e85baccf77cd60875ba 2024-11-08T00:25:02,826 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/a9a70570fd034e85baccf77cd60875ba, entries=150, sequenceid=616, filesize=12.0 K 2024-11-08T00:25:02,827 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=20.13 KB/20610 for 94c140d3911f03ee9ce6997a067b1a83 in 1279ms, sequenceid=616, compaction requested=true 2024-11-08T00:25:02,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:25:02,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c140d3911f03ee9ce6997a067b1a83:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T00:25:02,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:25:02,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c140d3911f03ee9ce6997a067b1a83:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T00:25:02,827 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:25:02,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:25:02,827 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:25:02,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 94c140d3911f03ee9ce6997a067b1a83:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:25:02,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:25:02,828 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38265 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:25:02,828 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38265 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:25:02,828 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): 94c140d3911f03ee9ce6997a067b1a83/B is initiating minor compaction (all files) 2024-11-08T00:25:02,828 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): 94c140d3911f03ee9ce6997a067b1a83/A is initiating minor compaction (all files) 2024-11-08T00:25:02,828 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c140d3911f03ee9ce6997a067b1a83/B in TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:25:02,828 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c140d3911f03ee9ce6997a067b1a83/A in TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:25:02,828 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/5bc479e62b274c05bb7af58a1aa6f9b2, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/99e0deaac2724831ab3e5bbb9ffb9e67, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/363342f9d45a4c79b84609f8fdbdfa47] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp, totalSize=37.4 K 2024-11-08T00:25:02,828 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/4682bf4aef774af29acc05a94b80b21d, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/1e9f1c3027b34c6fb05d71e064b4b992, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/595e8ed355664677a75b4bf4fd66b16e] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp, totalSize=37.4 K 2024-11-08T00:25:02,828 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bc479e62b274c05bb7af58a1aa6f9b2, keycount=150, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=576, earliestPutTs=1731025499345 2024-11-08T00:25:02,828 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4682bf4aef774af29acc05a94b80b21d, keycount=150, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=576, earliestPutTs=1731025499345 2024-11-08T00:25:02,829 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e9f1c3027b34c6fb05d71e064b4b992, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=595, earliestPutTs=1731025499939 2024-11-08T00:25:02,829 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 99e0deaac2724831ab3e5bbb9ffb9e67, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=595, earliestPutTs=1731025499939 2024-11-08T00:25:02,829 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 595e8ed355664677a75b4bf4fd66b16e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=616, earliestPutTs=1731025500379 2024-11-08T00:25:02,829 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 363342f9d45a4c79b84609f8fdbdfa47, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=616, earliestPutTs=1731025500379 2024-11-08T00:25:02,836 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c140d3911f03ee9ce6997a067b1a83#B#compaction#314 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:25:02,837 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/973e15fb62014ae69526eb67399740e4 is 50, key is test_row_0/B:col10/1731025500398/Put/seqid=0 2024-11-08T00:25:02,853 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c140d3911f03ee9ce6997a067b1a83#A#compaction#315 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:25:02,854 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/0ec640867a1643ed9e25c71a314cdd14 is 50, key is test_row_0/A:col10/1731025500398/Put/seqid=0 2024-11-08T00:25:02,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742187_1363 (size=13765) 2024-11-08T00:25:02,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742186_1362 (size=13765) 2024-11-08T00:25:02,918 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T00:25:03,286 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/0ec640867a1643ed9e25c71a314cdd14 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/0ec640867a1643ed9e25c71a314cdd14 2024-11-08T00:25:03,288 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/973e15fb62014ae69526eb67399740e4 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/973e15fb62014ae69526eb67399740e4 2024-11-08T00:25:03,292 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94c140d3911f03ee9ce6997a067b1a83/A of 94c140d3911f03ee9ce6997a067b1a83 into 0ec640867a1643ed9e25c71a314cdd14(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:25:03,292 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94c140d3911f03ee9ce6997a067b1a83/B of 94c140d3911f03ee9ce6997a067b1a83 into 973e15fb62014ae69526eb67399740e4(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:25:03,292 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:25:03,292 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:25:03,292 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., storeName=94c140d3911f03ee9ce6997a067b1a83/B, priority=13, startTime=1731025502827; duration=0sec 2024-11-08T00:25:03,292 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., storeName=94c140d3911f03ee9ce6997a067b1a83/A, priority=13, startTime=1731025502827; duration=0sec 2024-11-08T00:25:03,292 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:25:03,292 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c140d3911f03ee9ce6997a067b1a83:A 2024-11-08T00:25:03,292 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:25:03,292 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c140d3911f03ee9ce6997a067b1a83:B 2024-11-08T00:25:03,292 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:25:03,293 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38231 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:25:03,293 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): 94c140d3911f03ee9ce6997a067b1a83/C is initiating minor compaction (all files) 2024-11-08T00:25:03,293 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 94c140d3911f03ee9ce6997a067b1a83/C in TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:25:03,293 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/ee81d2d99ddb4603a28e65162d42323b, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/f599f458dfed4a83a548673977854a1e, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/a9a70570fd034e85baccf77cd60875ba] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp, totalSize=37.3 K 2024-11-08T00:25:03,294 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee81d2d99ddb4603a28e65162d42323b, keycount=150, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=576, earliestPutTs=1731025499345 2024-11-08T00:25:03,294 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting f599f458dfed4a83a548673977854a1e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=595, earliestPutTs=1731025499939 2024-11-08T00:25:03,294 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9a70570fd034e85baccf77cd60875ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=616, earliestPutTs=1731025500379 2024-11-08T00:25:03,309 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 94c140d3911f03ee9ce6997a067b1a83#C#compaction#316 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:25:03,310 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/3bdb67653af349508fbc055a68bc0337 is 50, key is test_row_0/C:col10/1731025500398/Put/seqid=0 2024-11-08T00:25:03,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742188_1364 (size=13731) 2024-11-08T00:25:03,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-08T00:25:03,377 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:25:03,721 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/3bdb67653af349508fbc055a68bc0337 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/3bdb67653af349508fbc055a68bc0337 2024-11-08T00:25:03,725 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 94c140d3911f03ee9ce6997a067b1a83/C of 94c140d3911f03ee9ce6997a067b1a83 into 3bdb67653af349508fbc055a68bc0337(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:25:03,726 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 94c140d3911f03ee9ce6997a067b1a83: 2024-11-08T00:25:03,726 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., storeName=94c140d3911f03ee9ce6997a067b1a83/C, priority=13, startTime=1731025502827; duration=0sec 2024-11-08T00:25:03,726 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:25:03,726 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 94c140d3911f03ee9ce6997a067b1a83:C 2024-11-08T00:25:07,719 INFO [Thread-1093 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1093. 2024-11-08T00:25:07,719 DEBUG [Thread-1093 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T00:25:07,719 DEBUG [Thread-1093 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:07,719 DEBUG [Thread-1093 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:07,719 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:25:07,719 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-08T00:25:07,719 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 107 2024-11-08T00:25:07,719 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-08T00:25:07,719 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 97 2024-11-08T00:25:07,719 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 103 2024-11-08T00:25:07,719 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 114 2024-11-08T00:25:07,719 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-08T00:25:07,719 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2541 2024-11-08T00:25:07,719 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2559 2024-11-08T00:25:07,719 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2447 2024-11-08T00:25:07,719 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2544 2024-11-08T00:25:07,719 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2589 2024-11-08T00:25:07,719 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-08T00:25:07,719 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T00:25:07,719 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T00:25:07,720 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:07,720 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:07,720 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:25:07,721 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-08T00:25:07,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-08T00:25:07,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-08T00:25:07,724 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731025507724"}]},"ts":"1731025507724"} 2024-11-08T00:25:07,726 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-08T00:25:07,726 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-08T00:25:07,726 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-08T00:25:07,728 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=94c140d3911f03ee9ce6997a067b1a83, UNASSIGN}] 2024-11-08T00:25:07,729 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=94c140d3911f03ee9ce6997a067b1a83, UNASSIGN 2024-11-08T00:25:07,729 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=94c140d3911f03ee9ce6997a067b1a83, regionState=CLOSING, regionLocation=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:07,731 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=94c140d3911f03ee9ce6997a067b1a83, UNASSIGN because future has completed 2024-11-08T00:25:07,731 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-08T00:25:07,731 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure 94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856}] 2024-11-08T00:25:07,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-08T00:25:07,884 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(122): Close 94c140d3911f03ee9ce6997a067b1a83 2024-11-08T00:25:07,884 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-08T00:25:07,884 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1722): Closing 94c140d3911f03ee9ce6997a067b1a83, disabling compactions & flushes 2024-11-08T00:25:07,884 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:25:07,884 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:25:07,884 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. after waiting 0 ms 2024-11-08T00:25:07,884 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:25:07,884 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(2902): Flushing 94c140d3911f03ee9ce6997a067b1a83 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-08T00:25:07,890 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/4bcc9baa6f3e4197b243d2954e0371b4 is 50, key is test_row_0/A:col10/1731025501567/Put/seqid=0 2024-11-08T00:25:07,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742189_1365 (size=12301) 2024-11-08T00:25:07,897 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=626 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/4bcc9baa6f3e4197b243d2954e0371b4 2024-11-08T00:25:07,905 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/e7c33462ac0f42e39a31a55e75602ea1 is 50, key is test_row_0/B:col10/1731025501567/Put/seqid=0 2024-11-08T00:25:07,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742190_1366 (size=12301) 2024-11-08T00:25:07,913 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=626 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/e7c33462ac0f42e39a31a55e75602ea1 2024-11-08T00:25:07,921 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/5dc60004b0864c4d996a92eaadc852fc is 50, key is test_row_0/C:col10/1731025501567/Put/seqid=0 2024-11-08T00:25:07,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742191_1367 (size=12301) 2024-11-08T00:25:08,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-08T00:25:08,336 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=626 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/5dc60004b0864c4d996a92eaadc852fc 2024-11-08T00:25:08,341 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/A/4bcc9baa6f3e4197b243d2954e0371b4 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/4bcc9baa6f3e4197b243d2954e0371b4 2024-11-08T00:25:08,344 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/4bcc9baa6f3e4197b243d2954e0371b4, entries=150, sequenceid=626, filesize=12.0 K 2024-11-08T00:25:08,345 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/B/e7c33462ac0f42e39a31a55e75602ea1 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/e7c33462ac0f42e39a31a55e75602ea1 2024-11-08T00:25:08,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-08T00:25:08,350 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/e7c33462ac0f42e39a31a55e75602ea1, entries=150, sequenceid=626, filesize=12.0 K 2024-11-08T00:25:08,350 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/.tmp/C/5dc60004b0864c4d996a92eaadc852fc as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/5dc60004b0864c4d996a92eaadc852fc 2024-11-08T00:25:08,355 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/5dc60004b0864c4d996a92eaadc852fc, entries=150, sequenceid=626, filesize=12.0 K 2024-11-08T00:25:08,356 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 94c140d3911f03ee9ce6997a067b1a83 in 472ms, sequenceid=626, compaction requested=false 2024-11-08T00:25:08,356 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/2fe06076275f4ea3833fad859f37d61a, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/320dd2f2da8d498dad74c26cbd0fdaf9, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/c8463c6a7b604d5784946940fe0c34a3, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/af40171d81d648788480eb61db332c79, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/fb939850917e458fa0eef9525ff7b74a, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/098d51f45b174fc298852c998b3b7687, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/2d23523cc58342709ad8e73ef4657b6a, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/d669030354e24066bb345b6d0a8e96f2, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/3367ec7cf21b41c4a0d32d8fc8f2eb79, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/88294393938942c388fd8edf9ae8a9b3, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/1b0f19733e2b4e79bce06a36afab9eb7, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/ac62232696c9411ea2bf66e56573ae1c, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/dbd7b3c6b32b408c9417049cf57ecd8b, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/25a375c163f244e8ad3f0146e3924841, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/22c8b4063bee4cadbadc52d8b1908642, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/8bb9bd8da7c045d7a5fb11cba35bd530, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/4bc09d38c8654924859dea99608a9c08, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/95128cd9c9684670a2f38ffe2269802d, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/2ab8a02378eb447f8bedd75dad2f87f3, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/525ef286526f4d53bd6d0e975f91a881, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/359eed3ddd7c41a28051ae7d3f09f79d, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/01f1031e07f4400d91504888ea614f56, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/581a8d2cac7949c883d7c81731c7ec36, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/22bd69856e984e16b9e96ed452ba2d44, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/40776fa00c0144c0b471cb0008ce4b4f, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/5b00d2154e064af29355c56ec9fc9dd5, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/3c6adb5cdb614f7b9283affd8a1493ec, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/60111580c7f545e5a5c29d464e7e7324, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/7bba2f50971e42b7a896a04a997f090c, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/beb16b12b92b41f7be8321be4d12e20e, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/1e2b041e45874d70bbf66c0c0c0cbb1b, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/5684bfe09bcb412a9ff529bdd2ce2ee7, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/4675ed2c003947a89a9f847cc6b9b456, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/bfaed82349a54adc9bb250a2b10584fb, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/38d3d32440cb427eb5ac6ff587002b47, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/063a45b4bdfe42d5af282360da9ea617, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/e00aaf4124114beabf4f1ae17367dbbe, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/240dcb2e56524f0f86701e331e3c4753, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/80e65fd512fd45d5aacdd9c3bec4c2a6, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/4e9366be52134e40bdd0a4f0ba3c3fdf, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/4682bf4aef774af29acc05a94b80b21d, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/1e9f1c3027b34c6fb05d71e064b4b992, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/595e8ed355664677a75b4bf4fd66b16e] to archive 2024-11-08T00:25:08,358 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T00:25:08,361 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/2fe06076275f4ea3833fad859f37d61a to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/2fe06076275f4ea3833fad859f37d61a 2024-11-08T00:25:08,362 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/320dd2f2da8d498dad74c26cbd0fdaf9 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/320dd2f2da8d498dad74c26cbd0fdaf9 2024-11-08T00:25:08,364 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/c8463c6a7b604d5784946940fe0c34a3 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/c8463c6a7b604d5784946940fe0c34a3 2024-11-08T00:25:08,366 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/af40171d81d648788480eb61db332c79 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/af40171d81d648788480eb61db332c79 2024-11-08T00:25:08,367 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/fb939850917e458fa0eef9525ff7b74a to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/fb939850917e458fa0eef9525ff7b74a 2024-11-08T00:25:08,368 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/098d51f45b174fc298852c998b3b7687 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/098d51f45b174fc298852c998b3b7687 2024-11-08T00:25:08,370 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/2d23523cc58342709ad8e73ef4657b6a to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/2d23523cc58342709ad8e73ef4657b6a 2024-11-08T00:25:08,371 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/d669030354e24066bb345b6d0a8e96f2 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/d669030354e24066bb345b6d0a8e96f2 2024-11-08T00:25:08,372 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/3367ec7cf21b41c4a0d32d8fc8f2eb79 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/3367ec7cf21b41c4a0d32d8fc8f2eb79 2024-11-08T00:25:08,374 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/88294393938942c388fd8edf9ae8a9b3 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/88294393938942c388fd8edf9ae8a9b3 2024-11-08T00:25:08,375 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/1b0f19733e2b4e79bce06a36afab9eb7 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/1b0f19733e2b4e79bce06a36afab9eb7 2024-11-08T00:25:08,376 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/ac62232696c9411ea2bf66e56573ae1c to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/ac62232696c9411ea2bf66e56573ae1c 2024-11-08T00:25:08,378 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/dbd7b3c6b32b408c9417049cf57ecd8b to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/dbd7b3c6b32b408c9417049cf57ecd8b 2024-11-08T00:25:08,379 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/25a375c163f244e8ad3f0146e3924841 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/25a375c163f244e8ad3f0146e3924841 2024-11-08T00:25:08,381 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/22c8b4063bee4cadbadc52d8b1908642 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/22c8b4063bee4cadbadc52d8b1908642 2024-11-08T00:25:08,382 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/8bb9bd8da7c045d7a5fb11cba35bd530 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/8bb9bd8da7c045d7a5fb11cba35bd530 2024-11-08T00:25:08,383 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/4bc09d38c8654924859dea99608a9c08 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/4bc09d38c8654924859dea99608a9c08 2024-11-08T00:25:08,390 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/95128cd9c9684670a2f38ffe2269802d to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/95128cd9c9684670a2f38ffe2269802d 2024-11-08T00:25:08,392 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/2ab8a02378eb447f8bedd75dad2f87f3 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/2ab8a02378eb447f8bedd75dad2f87f3 2024-11-08T00:25:08,393 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/525ef286526f4d53bd6d0e975f91a881 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/525ef286526f4d53bd6d0e975f91a881 2024-11-08T00:25:08,395 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/359eed3ddd7c41a28051ae7d3f09f79d to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/359eed3ddd7c41a28051ae7d3f09f79d 2024-11-08T00:25:08,396 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/01f1031e07f4400d91504888ea614f56 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/01f1031e07f4400d91504888ea614f56 2024-11-08T00:25:08,398 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/581a8d2cac7949c883d7c81731c7ec36 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/581a8d2cac7949c883d7c81731c7ec36 2024-11-08T00:25:08,399 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/22bd69856e984e16b9e96ed452ba2d44 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/22bd69856e984e16b9e96ed452ba2d44 2024-11-08T00:25:08,401 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/40776fa00c0144c0b471cb0008ce4b4f to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/40776fa00c0144c0b471cb0008ce4b4f 2024-11-08T00:25:08,403 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/5b00d2154e064af29355c56ec9fc9dd5 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/5b00d2154e064af29355c56ec9fc9dd5 2024-11-08T00:25:08,404 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/3c6adb5cdb614f7b9283affd8a1493ec to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/3c6adb5cdb614f7b9283affd8a1493ec 2024-11-08T00:25:08,405 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/60111580c7f545e5a5c29d464e7e7324 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/60111580c7f545e5a5c29d464e7e7324 2024-11-08T00:25:08,407 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/7bba2f50971e42b7a896a04a997f090c to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/7bba2f50971e42b7a896a04a997f090c 2024-11-08T00:25:08,409 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/beb16b12b92b41f7be8321be4d12e20e to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/beb16b12b92b41f7be8321be4d12e20e 2024-11-08T00:25:08,411 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/1e2b041e45874d70bbf66c0c0c0cbb1b to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/1e2b041e45874d70bbf66c0c0c0cbb1b 2024-11-08T00:25:08,412 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/5684bfe09bcb412a9ff529bdd2ce2ee7 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/5684bfe09bcb412a9ff529bdd2ce2ee7 2024-11-08T00:25:08,413 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/4675ed2c003947a89a9f847cc6b9b456 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/4675ed2c003947a89a9f847cc6b9b456 2024-11-08T00:25:08,415 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/bfaed82349a54adc9bb250a2b10584fb to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/bfaed82349a54adc9bb250a2b10584fb 2024-11-08T00:25:08,416 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/38d3d32440cb427eb5ac6ff587002b47 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/38d3d32440cb427eb5ac6ff587002b47 2024-11-08T00:25:08,418 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/063a45b4bdfe42d5af282360da9ea617 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/063a45b4bdfe42d5af282360da9ea617 2024-11-08T00:25:08,420 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/e00aaf4124114beabf4f1ae17367dbbe to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/e00aaf4124114beabf4f1ae17367dbbe 2024-11-08T00:25:08,421 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/240dcb2e56524f0f86701e331e3c4753 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/240dcb2e56524f0f86701e331e3c4753 2024-11-08T00:25:08,423 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/80e65fd512fd45d5aacdd9c3bec4c2a6 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/80e65fd512fd45d5aacdd9c3bec4c2a6 2024-11-08T00:25:08,424 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/4e9366be52134e40bdd0a4f0ba3c3fdf to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/4e9366be52134e40bdd0a4f0ba3c3fdf 2024-11-08T00:25:08,429 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/4682bf4aef774af29acc05a94b80b21d to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/4682bf4aef774af29acc05a94b80b21d 2024-11-08T00:25:08,430 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/1e9f1c3027b34c6fb05d71e064b4b992 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/1e9f1c3027b34c6fb05d71e064b4b992 2024-11-08T00:25:08,432 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/595e8ed355664677a75b4bf4fd66b16e to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/595e8ed355664677a75b4bf4fd66b16e 2024-11-08T00:25:08,436 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/7eaf26e8ac774e5a9f1df20ed379921c, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/ec65a3f81da148bd90fcce853677dde3, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/a8427baf431745148d5852097d517d01, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/680c0d37aff34c15a7110e6de002d5c0, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/b058f83457784463978b70d227c0de6c, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/bd40ca65994e437081f2529e4862f983, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/3c5f4cfdb6504851b4817986047095c7, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/ec27786c9ec045bab417e4fb5a59a844, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/5731bf186df04e6aa4ab467282785238, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/9770c53929be45ce975bae97c8bbc39f, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/9f298e83c3cb42c98abae77d98309a07, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/7bfe37849c6744a7bedc8880e69d4c5d, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/d41a21fb56104aaba98b82868179bfef, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/9715a1ad407648778dc152f0b7362dd7, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/46d937ee9f4e4b2f86fd7ed47d91d274, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/e1d4fd9c50cd4918ad1924bb3627e80a, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/b44ad54bba97499bba82e7248f55fc74, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/2b7a9e9d2ca5447ea13eb39151206822, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/45b55ac89efe4221bf83a1747aada78d, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/99c8524e26b44e928b8874ef89489786, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/008cbdaba15544258a40ee6966bb01e9, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/457a985ce6c34bc3b793e39da3d5d421, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/8bf9246939b946ae9e2b15aceb15c0a1, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/1bd56d42bf44433d84024cf406a31b31, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/be4830226c41488b989bcfcaa60888c8, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/ae0c17d2e053464ba6410bbd6d98d6ba, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/18a1a997cc724d4aa78b82de804f58ac, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/b45e45c889ca45c1bf14521a41adbe43, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/672f6bb0d7594c9285311d5bcf19b0a0, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/be27ea9bd5b846f8b2cd682caba7beb6, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/298ae7cb318f4608a729c3185d167b3b, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/ef0a39c55d1d42e4942e2570af10f8b8, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/8a70e76d01934e628395eeb165ea7fde, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/855b06d275a74b2f9c0d7fa283e1c6d4, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/1710ded12afb4a9c8fb9bf7f85e081f6, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/556e07fb27d74523ab33fd82e973268a, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/213cfd4dddfa4b35aac514102cda043a, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/424325aa8df4435d94750b10917ef85d, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/6634a4337da14bc490cdff5f1c465015, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/5bc479e62b274c05bb7af58a1aa6f9b2, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/0fc0325f83b04cdaa1217a94ce583c1f, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/99e0deaac2724831ab3e5bbb9ffb9e67, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/363342f9d45a4c79b84609f8fdbdfa47] to archive 2024-11-08T00:25:08,437 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T00:25:08,439 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/7eaf26e8ac774e5a9f1df20ed379921c to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/7eaf26e8ac774e5a9f1df20ed379921c 2024-11-08T00:25:08,440 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/ec65a3f81da148bd90fcce853677dde3 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/ec65a3f81da148bd90fcce853677dde3 2024-11-08T00:25:08,443 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/a8427baf431745148d5852097d517d01 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/a8427baf431745148d5852097d517d01 2024-11-08T00:25:08,445 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/680c0d37aff34c15a7110e6de002d5c0 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/680c0d37aff34c15a7110e6de002d5c0 2024-11-08T00:25:08,446 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/b058f83457784463978b70d227c0de6c to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/b058f83457784463978b70d227c0de6c 2024-11-08T00:25:08,448 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/bd40ca65994e437081f2529e4862f983 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/bd40ca65994e437081f2529e4862f983 2024-11-08T00:25:08,449 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/3c5f4cfdb6504851b4817986047095c7 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/3c5f4cfdb6504851b4817986047095c7 2024-11-08T00:25:08,449 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/ec27786c9ec045bab417e4fb5a59a844 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/ec27786c9ec045bab417e4fb5a59a844 2024-11-08T00:25:08,451 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/5731bf186df04e6aa4ab467282785238 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/5731bf186df04e6aa4ab467282785238 2024-11-08T00:25:08,453 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/9770c53929be45ce975bae97c8bbc39f to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/9770c53929be45ce975bae97c8bbc39f 2024-11-08T00:25:08,454 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/9f298e83c3cb42c98abae77d98309a07 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/9f298e83c3cb42c98abae77d98309a07 2024-11-08T00:25:08,456 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/7bfe37849c6744a7bedc8880e69d4c5d to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/7bfe37849c6744a7bedc8880e69d4c5d 2024-11-08T00:25:08,457 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/d41a21fb56104aaba98b82868179bfef to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/d41a21fb56104aaba98b82868179bfef 2024-11-08T00:25:08,458 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/9715a1ad407648778dc152f0b7362dd7 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/9715a1ad407648778dc152f0b7362dd7 2024-11-08T00:25:08,459 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/46d937ee9f4e4b2f86fd7ed47d91d274 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/46d937ee9f4e4b2f86fd7ed47d91d274 2024-11-08T00:25:08,460 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/e1d4fd9c50cd4918ad1924bb3627e80a to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/e1d4fd9c50cd4918ad1924bb3627e80a 2024-11-08T00:25:08,461 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/b44ad54bba97499bba82e7248f55fc74 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/b44ad54bba97499bba82e7248f55fc74 2024-11-08T00:25:08,462 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/2b7a9e9d2ca5447ea13eb39151206822 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/2b7a9e9d2ca5447ea13eb39151206822 2024-11-08T00:25:08,463 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/45b55ac89efe4221bf83a1747aada78d to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/45b55ac89efe4221bf83a1747aada78d 2024-11-08T00:25:08,464 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/99c8524e26b44e928b8874ef89489786 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/99c8524e26b44e928b8874ef89489786 2024-11-08T00:25:08,465 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/008cbdaba15544258a40ee6966bb01e9 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/008cbdaba15544258a40ee6966bb01e9 2024-11-08T00:25:08,466 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/457a985ce6c34bc3b793e39da3d5d421 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/457a985ce6c34bc3b793e39da3d5d421 2024-11-08T00:25:08,467 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/8bf9246939b946ae9e2b15aceb15c0a1 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/8bf9246939b946ae9e2b15aceb15c0a1 2024-11-08T00:25:08,467 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/1bd56d42bf44433d84024cf406a31b31 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/1bd56d42bf44433d84024cf406a31b31 2024-11-08T00:25:08,468 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/be4830226c41488b989bcfcaa60888c8 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/be4830226c41488b989bcfcaa60888c8 2024-11-08T00:25:08,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/ae0c17d2e053464ba6410bbd6d98d6ba to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/ae0c17d2e053464ba6410bbd6d98d6ba 2024-11-08T00:25:08,470 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/18a1a997cc724d4aa78b82de804f58ac to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/18a1a997cc724d4aa78b82de804f58ac 2024-11-08T00:25:08,478 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/b45e45c889ca45c1bf14521a41adbe43 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/b45e45c889ca45c1bf14521a41adbe43 2024-11-08T00:25:08,480 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/672f6bb0d7594c9285311d5bcf19b0a0 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/672f6bb0d7594c9285311d5bcf19b0a0 2024-11-08T00:25:08,481 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/be27ea9bd5b846f8b2cd682caba7beb6 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/be27ea9bd5b846f8b2cd682caba7beb6 2024-11-08T00:25:08,482 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/298ae7cb318f4608a729c3185d167b3b to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/298ae7cb318f4608a729c3185d167b3b 2024-11-08T00:25:08,483 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/ef0a39c55d1d42e4942e2570af10f8b8 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/ef0a39c55d1d42e4942e2570af10f8b8 2024-11-08T00:25:08,484 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/8a70e76d01934e628395eeb165ea7fde to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/8a70e76d01934e628395eeb165ea7fde 2024-11-08T00:25:08,485 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/855b06d275a74b2f9c0d7fa283e1c6d4 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/855b06d275a74b2f9c0d7fa283e1c6d4 2024-11-08T00:25:08,486 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/1710ded12afb4a9c8fb9bf7f85e081f6 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/1710ded12afb4a9c8fb9bf7f85e081f6 2024-11-08T00:25:08,487 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/556e07fb27d74523ab33fd82e973268a to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/556e07fb27d74523ab33fd82e973268a 2024-11-08T00:25:08,488 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/213cfd4dddfa4b35aac514102cda043a to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/213cfd4dddfa4b35aac514102cda043a 2024-11-08T00:25:08,489 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/424325aa8df4435d94750b10917ef85d to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/424325aa8df4435d94750b10917ef85d 2024-11-08T00:25:08,490 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/6634a4337da14bc490cdff5f1c465015 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/6634a4337da14bc490cdff5f1c465015 2024-11-08T00:25:08,491 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/5bc479e62b274c05bb7af58a1aa6f9b2 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/5bc479e62b274c05bb7af58a1aa6f9b2 2024-11-08T00:25:08,492 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/0fc0325f83b04cdaa1217a94ce583c1f to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/0fc0325f83b04cdaa1217a94ce583c1f 2024-11-08T00:25:08,493 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/99e0deaac2724831ab3e5bbb9ffb9e67 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/99e0deaac2724831ab3e5bbb9ffb9e67 2024-11-08T00:25:08,494 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/363342f9d45a4c79b84609f8fdbdfa47 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/363342f9d45a4c79b84609f8fdbdfa47 2024-11-08T00:25:08,496 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/a3dea4bfe9924640972616982fae704d, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/0b828357afdb4084a5d5105ac0c4f9b5, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/e781e280643642aba38501895335e94f, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/e682d05e77a24a7c88bbeaf9ee5614f3, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/11469dca7a0d48f5a84d981160702c01, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/c2a6d3a502ad42978f4f5a5529af7c76, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/291104c537dc44dcbc33d952254d7804, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/47db12face5a44d0ba3853429827ef92, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/bef16efff120483096c099e8c426d45a, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/4af8202f9ae6413cab00e4f155a6a67a, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/86dfa5eb208b460eb9a80641a7931538, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/a154f2b5909f4e04bc0c31cf8f977942, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/4bcd6a670e914dba814e7927de6d3852, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/53f48659d7aa4a4d9535a91beba306e9, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/48773eb5c0db402bb5e5828775fb2a61, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/0af7a379dd804196ade4bf97a9ff1686, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/0c13273098c74711a8cb7b25311cfdc1, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/49fd83d260624c0585fe344a52bd09cf, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/68c6531d0ebf46bf9291b04707fdf99b, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/60e0fdef436042c184a9a89cd1838419, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/9626c3067d5240059e8ba479bb0b74af, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/523fcae6f9a64ee7a5067dc91d5f04c1, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/6678a2a199a948e499d663c93ab95496, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/49e98abd406045d583b000d028570898, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/3124bd10656c436f963b8e0c6a207f37, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/ab7ff9552c644ba1b10f0af075cd19de, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/b3bfd60cbc764057ac2dfce140533c21, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/5dd310ccc61d41f480154c9fbe7c8a0f, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/0808968364b04bedbfa6b574723184de, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/23673bab20584c379061f6c07f0dcb30, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/2d61f78fe9ed46469851a0ea4c30ab56, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/1db6c23ef4f84660831424741515a5a0, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/1c8fc20bf7f34952a7cd72c00861e9ec, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/2c1b2deaf60749089b1a2a44ed3172ea, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/f8233d534b8f4f208928dcc9ddc0d827, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/f8eef3d328c74293a17d18312e1e1942, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/f531e471711942ef82eccd33b081e1c0, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/498c204495f7434db91f7ea78b009ce0, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/ee81d2d99ddb4603a28e65162d42323b, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/b8d7fadd8ff54d82b198445659f68f1e, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/f599f458dfed4a83a548673977854a1e, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/a9a70570fd034e85baccf77cd60875ba] to archive 2024-11-08T00:25:08,497 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T00:25:08,498 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/a3dea4bfe9924640972616982fae704d to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/a3dea4bfe9924640972616982fae704d 2024-11-08T00:25:08,499 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/0b828357afdb4084a5d5105ac0c4f9b5 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/0b828357afdb4084a5d5105ac0c4f9b5 2024-11-08T00:25:08,500 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/e781e280643642aba38501895335e94f to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/e781e280643642aba38501895335e94f 2024-11-08T00:25:08,502 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/e682d05e77a24a7c88bbeaf9ee5614f3 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/e682d05e77a24a7c88bbeaf9ee5614f3 2024-11-08T00:25:08,503 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/11469dca7a0d48f5a84d981160702c01 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/11469dca7a0d48f5a84d981160702c01 2024-11-08T00:25:08,504 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/c2a6d3a502ad42978f4f5a5529af7c76 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/c2a6d3a502ad42978f4f5a5529af7c76 2024-11-08T00:25:08,505 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/291104c537dc44dcbc33d952254d7804 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/291104c537dc44dcbc33d952254d7804 2024-11-08T00:25:08,506 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/47db12face5a44d0ba3853429827ef92 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/47db12face5a44d0ba3853429827ef92 2024-11-08T00:25:08,507 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/bef16efff120483096c099e8c426d45a to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/bef16efff120483096c099e8c426d45a 2024-11-08T00:25:08,508 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/4af8202f9ae6413cab00e4f155a6a67a to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/4af8202f9ae6413cab00e4f155a6a67a 2024-11-08T00:25:08,510 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/86dfa5eb208b460eb9a80641a7931538 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/86dfa5eb208b460eb9a80641a7931538 2024-11-08T00:25:08,511 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/a154f2b5909f4e04bc0c31cf8f977942 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/a154f2b5909f4e04bc0c31cf8f977942 2024-11-08T00:25:08,512 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/4bcd6a670e914dba814e7927de6d3852 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/4bcd6a670e914dba814e7927de6d3852 2024-11-08T00:25:08,521 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/53f48659d7aa4a4d9535a91beba306e9 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/53f48659d7aa4a4d9535a91beba306e9 2024-11-08T00:25:08,522 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/48773eb5c0db402bb5e5828775fb2a61 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/48773eb5c0db402bb5e5828775fb2a61 2024-11-08T00:25:08,523 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/0af7a379dd804196ade4bf97a9ff1686 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/0af7a379dd804196ade4bf97a9ff1686 2024-11-08T00:25:08,525 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/0c13273098c74711a8cb7b25311cfdc1 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/0c13273098c74711a8cb7b25311cfdc1 2024-11-08T00:25:08,526 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/49fd83d260624c0585fe344a52bd09cf to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/49fd83d260624c0585fe344a52bd09cf 2024-11-08T00:25:08,527 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/68c6531d0ebf46bf9291b04707fdf99b to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/68c6531d0ebf46bf9291b04707fdf99b 2024-11-08T00:25:08,527 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/60e0fdef436042c184a9a89cd1838419 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/60e0fdef436042c184a9a89cd1838419 2024-11-08T00:25:08,528 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/9626c3067d5240059e8ba479bb0b74af to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/9626c3067d5240059e8ba479bb0b74af 2024-11-08T00:25:08,529 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/523fcae6f9a64ee7a5067dc91d5f04c1 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/523fcae6f9a64ee7a5067dc91d5f04c1 2024-11-08T00:25:08,530 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/6678a2a199a948e499d663c93ab95496 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/6678a2a199a948e499d663c93ab95496 2024-11-08T00:25:08,531 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/49e98abd406045d583b000d028570898 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/49e98abd406045d583b000d028570898 2024-11-08T00:25:08,532 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/3124bd10656c436f963b8e0c6a207f37 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/3124bd10656c436f963b8e0c6a207f37 2024-11-08T00:25:08,534 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/ab7ff9552c644ba1b10f0af075cd19de to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/ab7ff9552c644ba1b10f0af075cd19de 2024-11-08T00:25:08,535 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/b3bfd60cbc764057ac2dfce140533c21 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/b3bfd60cbc764057ac2dfce140533c21 2024-11-08T00:25:08,536 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/5dd310ccc61d41f480154c9fbe7c8a0f to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/5dd310ccc61d41f480154c9fbe7c8a0f 2024-11-08T00:25:08,537 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/0808968364b04bedbfa6b574723184de to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/0808968364b04bedbfa6b574723184de 2024-11-08T00:25:08,538 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/23673bab20584c379061f6c07f0dcb30 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/23673bab20584c379061f6c07f0dcb30 2024-11-08T00:25:08,540 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/2d61f78fe9ed46469851a0ea4c30ab56 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/2d61f78fe9ed46469851a0ea4c30ab56 2024-11-08T00:25:08,540 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/1db6c23ef4f84660831424741515a5a0 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/1db6c23ef4f84660831424741515a5a0 2024-11-08T00:25:08,541 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/1c8fc20bf7f34952a7cd72c00861e9ec to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/1c8fc20bf7f34952a7cd72c00861e9ec 2024-11-08T00:25:08,542 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/2c1b2deaf60749089b1a2a44ed3172ea to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/2c1b2deaf60749089b1a2a44ed3172ea 2024-11-08T00:25:08,543 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/f8233d534b8f4f208928dcc9ddc0d827 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/f8233d534b8f4f208928dcc9ddc0d827 2024-11-08T00:25:08,544 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/f8eef3d328c74293a17d18312e1e1942 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/f8eef3d328c74293a17d18312e1e1942 2024-11-08T00:25:08,545 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/f531e471711942ef82eccd33b081e1c0 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/f531e471711942ef82eccd33b081e1c0 2024-11-08T00:25:08,545 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/498c204495f7434db91f7ea78b009ce0 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/498c204495f7434db91f7ea78b009ce0 2024-11-08T00:25:08,546 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/ee81d2d99ddb4603a28e65162d42323b to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/ee81d2d99ddb4603a28e65162d42323b 2024-11-08T00:25:08,547 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/b8d7fadd8ff54d82b198445659f68f1e to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/b8d7fadd8ff54d82b198445659f68f1e 2024-11-08T00:25:08,548 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/f599f458dfed4a83a548673977854a1e to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/f599f458dfed4a83a548673977854a1e 2024-11-08T00:25:08,548 DEBUG [StoreCloser-TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/a9a70570fd034e85baccf77cd60875ba to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/a9a70570fd034e85baccf77cd60875ba 2024-11-08T00:25:08,554 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/recovered.edits/629.seqid, newMaxSeqId=629, maxSeqId=1 2024-11-08T00:25:08,555 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. 2024-11-08T00:25:08,555 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1676): Region close journal for 94c140d3911f03ee9ce6997a067b1a83: Waiting for close lock at 1731025507884Running coprocessor pre-close hooks at 1731025507884Disabling compacts and flushes for region at 1731025507884Disabling writes for close at 1731025507884Obtaining lock to block concurrent updates at 1731025507884Preparing flush snapshotting stores in 94c140d3911f03ee9ce6997a067b1a83 at 1731025507884Finished memstore snapshotting TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1731025507885 (+1 ms)Flushing stores of TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83. at 1731025507885Flushing 94c140d3911f03ee9ce6997a067b1a83/A: creating writer at 1731025507885Flushing 94c140d3911f03ee9ce6997a067b1a83/A: appending metadata at 1731025507889 (+4 ms)Flushing 94c140d3911f03ee9ce6997a067b1a83/A: closing flushed file at 1731025507889Flushing 94c140d3911f03ee9ce6997a067b1a83/B: creating writer at 1731025507901 (+12 ms)Flushing 94c140d3911f03ee9ce6997a067b1a83/B: appending metadata at 1731025507904 (+3 ms)Flushing 94c140d3911f03ee9ce6997a067b1a83/B: closing flushed file at 1731025507905 (+1 ms)Flushing 94c140d3911f03ee9ce6997a067b1a83/C: creating writer at 1731025507917 (+12 ms)Flushing 94c140d3911f03ee9ce6997a067b1a83/C: appending metadata at 1731025507920 (+3 ms)Flushing 94c140d3911f03ee9ce6997a067b1a83/C: closing flushed file at 1731025507920Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@25d55e03: reopening flushed file at 1731025508340 (+420 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4bfd0d08: reopening flushed file at 1731025508344 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7f0a0e83: reopening flushed file at 1731025508350 (+6 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 94c140d3911f03ee9ce6997a067b1a83 in 472ms, sequenceid=626, compaction requested=false at 1731025508356 (+6 ms)Writing region close event to WAL at 1731025508550 (+194 ms)Running coprocessor post-close hooks at 1731025508554 (+4 ms)Closed at 1731025508554 2024-11-08T00:25:08,556 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(157): Closed 94c140d3911f03ee9ce6997a067b1a83 2024-11-08T00:25:08,557 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=94c140d3911f03ee9ce6997a067b1a83, regionState=CLOSED 2024-11-08T00:25:08,560 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure 94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 because future has completed 2024-11-08T00:25:08,564 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-08T00:25:08,564 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseRegionProcedure 94c140d3911f03ee9ce6997a067b1a83, server=3302f0f507bd,40479,1731025415856 in 830 msec 2024-11-08T00:25:08,566 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81 2024-11-08T00:25:08,567 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=94c140d3911f03ee9ce6997a067b1a83, UNASSIGN in 836 msec 2024-11-08T00:25:08,568 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-08T00:25:08,569 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 841 msec 2024-11-08T00:25:08,570 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731025508570"}]},"ts":"1731025508570"} 2024-11-08T00:25:08,571 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-08T00:25:08,571 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-08T00:25:08,573 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 851 msec 2024-11-08T00:25:08,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-08T00:25:08,857 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:25:08,857 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-08T00:25:08,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T00:25:08,859 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=84, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T00:25:08,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-08T00:25:08,860 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=84, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T00:25:08,862 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83 2024-11-08T00:25:08,864 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A, FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B, FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C, FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/recovered.edits] 2024-11-08T00:25:08,866 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/0ec640867a1643ed9e25c71a314cdd14 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/0ec640867a1643ed9e25c71a314cdd14 2024-11-08T00:25:08,867 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/4bcc9baa6f3e4197b243d2954e0371b4 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/A/4bcc9baa6f3e4197b243d2954e0371b4 2024-11-08T00:25:08,869 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/973e15fb62014ae69526eb67399740e4 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/973e15fb62014ae69526eb67399740e4 2024-11-08T00:25:08,870 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/e7c33462ac0f42e39a31a55e75602ea1 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/B/e7c33462ac0f42e39a31a55e75602ea1 2024-11-08T00:25:08,872 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/3bdb67653af349508fbc055a68bc0337 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/3bdb67653af349508fbc055a68bc0337 2024-11-08T00:25:08,873 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/5dc60004b0864c4d996a92eaadc852fc to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/C/5dc60004b0864c4d996a92eaadc852fc 2024-11-08T00:25:08,875 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/recovered.edits/629.seqid to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83/recovered.edits/629.seqid 2024-11-08T00:25:08,876 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/94c140d3911f03ee9ce6997a067b1a83 2024-11-08T00:25:08,876 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-08T00:25:08,878 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=84, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T00:25:08,881 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-08T00:25:08,884 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-08T00:25:08,885 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=84, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T00:25:08,885 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-08T00:25:08,885 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731025508885"}]},"ts":"9223372036854775807"} 2024-11-08T00:25:08,887 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-08T00:25:08,887 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 94c140d3911f03ee9ce6997a067b1a83, NAME => 'TestAcidGuarantees,,1731025479630.94c140d3911f03ee9ce6997a067b1a83.', STARTKEY => '', ENDKEY => ''}] 2024-11-08T00:25:08,888 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-08T00:25:08,888 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731025508888"}]},"ts":"9223372036854775807"} 2024-11-08T00:25:08,890 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-08T00:25:08,891 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=84, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T00:25:08,892 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 33 msec 2024-11-08T00:25:08,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-08T00:25:08,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-08T00:25:08,970 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:25:08,977 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=239 (was 242), OpenFileDescriptor=455 (was 467), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=785 (was 721) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4397 (was 5079) 2024-11-08T00:25:08,985 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=239, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=785, ProcessCount=11, AvailableMemoryMB=4397 2024-11-08T00:25:08,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-08T00:25:08,987 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T00:25:08,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=85, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-08T00:25:08,989 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T00:25:08,989 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:08,990 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 85 2024-11-08T00:25:08,990 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T00:25:08,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-08T00:25:08,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742192_1368 (size=959) 2024-11-08T00:25:08,997 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f 2024-11-08T00:25:09,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742193_1369 (size=53) 2024-11-08T00:25:09,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-08T00:25:09,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-08T00:25:09,402 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:25:09,402 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 0d1e33fe1a366293d5e32ce198366168, disabling compactions & flushes 2024-11-08T00:25:09,402 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:09,402 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:09,402 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. after waiting 0 ms 2024-11-08T00:25:09,402 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:09,402 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:09,402 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0d1e33fe1a366293d5e32ce198366168: Waiting for close lock at 1731025509402Disabling compacts and flushes for region at 1731025509402Disabling writes for close at 1731025509402Writing region close event to WAL at 1731025509402Closed at 1731025509402 2024-11-08T00:25:09,403 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T00:25:09,404 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731025509403"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731025509403"}]},"ts":"1731025509403"} 2024-11-08T00:25:09,406 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-08T00:25:09,407 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T00:25:09,407 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731025509407"}]},"ts":"1731025509407"} 2024-11-08T00:25:09,408 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-08T00:25:09,409 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0d1e33fe1a366293d5e32ce198366168, ASSIGN}] 2024-11-08T00:25:09,410 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0d1e33fe1a366293d5e32ce198366168, ASSIGN 2024-11-08T00:25:09,411 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0d1e33fe1a366293d5e32ce198366168, ASSIGN; state=OFFLINE, location=3302f0f507bd,40479,1731025415856; forceNewPlan=false, retain=false 2024-11-08T00:25:09,561 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=0d1e33fe1a366293d5e32ce198366168, regionState=OPENING, regionLocation=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:09,563 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0d1e33fe1a366293d5e32ce198366168, ASSIGN because future has completed 2024-11-08T00:25:09,563 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856}] 2024-11-08T00:25:09,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-08T00:25:09,719 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:09,719 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7752): Opening region: {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} 2024-11-08T00:25:09,719 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:09,719 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:25:09,719 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7794): checking encryption for 0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:09,719 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7797): checking classloading for 0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:09,720 INFO [StoreOpener-0d1e33fe1a366293d5e32ce198366168-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:09,721 INFO [StoreOpener-0d1e33fe1a366293d5e32ce198366168-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d1e33fe1a366293d5e32ce198366168 columnFamilyName A 2024-11-08T00:25:09,721 DEBUG [StoreOpener-0d1e33fe1a366293d5e32ce198366168-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:09,722 INFO [StoreOpener-0d1e33fe1a366293d5e32ce198366168-1 {}] regionserver.HStore(327): Store=0d1e33fe1a366293d5e32ce198366168/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:25:09,722 INFO [StoreOpener-0d1e33fe1a366293d5e32ce198366168-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:09,723 INFO [StoreOpener-0d1e33fe1a366293d5e32ce198366168-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d1e33fe1a366293d5e32ce198366168 columnFamilyName B 2024-11-08T00:25:09,723 DEBUG [StoreOpener-0d1e33fe1a366293d5e32ce198366168-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:09,723 INFO [StoreOpener-0d1e33fe1a366293d5e32ce198366168-1 {}] regionserver.HStore(327): Store=0d1e33fe1a366293d5e32ce198366168/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:25:09,723 INFO [StoreOpener-0d1e33fe1a366293d5e32ce198366168-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:09,724 INFO [StoreOpener-0d1e33fe1a366293d5e32ce198366168-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d1e33fe1a366293d5e32ce198366168 columnFamilyName C 2024-11-08T00:25:09,724 DEBUG [StoreOpener-0d1e33fe1a366293d5e32ce198366168-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:09,728 INFO [StoreOpener-0d1e33fe1a366293d5e32ce198366168-1 {}] regionserver.HStore(327): Store=0d1e33fe1a366293d5e32ce198366168/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:25:09,728 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1038): replaying wal for 0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:09,729 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:09,730 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:09,733 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1048): stopping wal replay for 0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:09,733 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1060): Cleaning up temporary data for 0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:09,734 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-08T00:25:09,735 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1093): writing seq id for 0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:09,737 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T00:25:09,738 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1114): Opened 0d1e33fe1a366293d5e32ce198366168; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60291848, jitterRate=-0.10158145427703857}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-08T00:25:09,738 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:09,739 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1006): Region open journal for 0d1e33fe1a366293d5e32ce198366168: Running coprocessor pre-open hook at 1731025509719Writing region info on filesystem at 1731025509719Initializing all the Stores at 1731025509720 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731025509720Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731025509720Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731025509720Cleaning up temporary data from old regions at 1731025509733 (+13 ms)Running coprocessor post-open hooks at 1731025509738 (+5 ms)Region opened successfully at 1731025509739 (+1 ms) 2024-11-08T00:25:09,740 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., pid=87, masterSystemTime=1731025509715 2024-11-08T00:25:09,742 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:09,742 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:09,742 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=0d1e33fe1a366293d5e32ce198366168, regionState=OPEN, openSeqNum=2, regionLocation=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:09,744 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 because future has completed 2024-11-08T00:25:09,746 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-08T00:25:09,746 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; OpenRegionProcedure 0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 in 182 msec 2024-11-08T00:25:09,748 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-11-08T00:25:09,748 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0d1e33fe1a366293d5e32ce198366168, ASSIGN in 337 msec 2024-11-08T00:25:09,749 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T00:25:09,749 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731025509749"}]},"ts":"1731025509749"} 2024-11-08T00:25:09,751 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-08T00:25:09,752 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T00:25:09,754 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 765 msec 2024-11-08T00:25:10,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-08T00:25:10,128 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:25:10,129 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e587032, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:10,129 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:25:10,129 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:25:10,130 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:25:10,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:25:10,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:25:10,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@685dc8d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:10,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:25:10,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:25:10,131 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:10,131 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43366, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:25:10,132 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5af1b015, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:10,132 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:25:10,133 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:25:10,133 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:25:10,135 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47188, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:25:10,135 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-08T00:25:10,136 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 3302f0f507bd,36785,1731025414356 2024-11-08T00:25:10,137 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@66c36d69 2024-11-08T00:25:10,137 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-08T00:25:10,138 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43376, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-08T00:25:10,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-08T00:25:10,140 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T00:25:10,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-08T00:25:10,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742194_1370 (size=995) 2024-11-08T00:25:10,162 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-08T00:25:10,162 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-08T00:25:10,165 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-08T00:25:10,168 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0d1e33fe1a366293d5e32ce198366168, REOPEN/MOVE}] 2024-11-08T00:25:10,169 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0d1e33fe1a366293d5e32ce198366168, REOPEN/MOVE 2024-11-08T00:25:10,170 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=0d1e33fe1a366293d5e32ce198366168, regionState=CLOSING, regionLocation=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:10,173 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0d1e33fe1a366293d5e32ce198366168, REOPEN/MOVE because future has completed 2024-11-08T00:25:10,173 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-08T00:25:10,173 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856}] 2024-11-08T00:25:10,326 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(122): Close 0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:10,326 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-08T00:25:10,326 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1722): Closing 0d1e33fe1a366293d5e32ce198366168, disabling compactions & flushes 2024-11-08T00:25:10,326 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:10,326 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:10,326 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. after waiting 0 ms 2024-11-08T00:25:10,326 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:10,329 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T00:25:10,330 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:10,330 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1676): Region close journal for 0d1e33fe1a366293d5e32ce198366168: Waiting for close lock at 1731025510326Running coprocessor pre-close hooks at 1731025510326Disabling compacts and flushes for region at 1731025510326Disabling writes for close at 1731025510326Writing region close event to WAL at 1731025510326Running coprocessor post-close hooks at 1731025510330 (+4 ms)Closed at 1731025510330 2024-11-08T00:25:10,330 WARN [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionServer(3299): Not adding moved region record: 0d1e33fe1a366293d5e32ce198366168 to self. 2024-11-08T00:25:10,333 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(157): Closed 0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:10,333 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=0d1e33fe1a366293d5e32ce198366168, regionState=CLOSED 2024-11-08T00:25:10,335 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 because future has completed 2024-11-08T00:25:10,347 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-08T00:25:10,347 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; CloseRegionProcedure 0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 in 164 msec 2024-11-08T00:25:10,348 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0d1e33fe1a366293d5e32ce198366168, REOPEN/MOVE; state=CLOSED, location=3302f0f507bd,40479,1731025415856; forceNewPlan=false, retain=true 2024-11-08T00:25:10,499 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=0d1e33fe1a366293d5e32ce198366168, regionState=OPENING, regionLocation=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:10,500 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0d1e33fe1a366293d5e32ce198366168, REOPEN/MOVE because future has completed 2024-11-08T00:25:10,501 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856}] 2024-11-08T00:25:10,655 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:10,655 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7752): Opening region: {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} 2024-11-08T00:25:10,656 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:10,656 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:25:10,656 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7794): checking encryption for 0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:10,656 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7797): checking classloading for 0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:10,657 INFO [StoreOpener-0d1e33fe1a366293d5e32ce198366168-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:10,658 INFO [StoreOpener-0d1e33fe1a366293d5e32ce198366168-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d1e33fe1a366293d5e32ce198366168 columnFamilyName A 2024-11-08T00:25:10,659 DEBUG [StoreOpener-0d1e33fe1a366293d5e32ce198366168-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:10,659 INFO [StoreOpener-0d1e33fe1a366293d5e32ce198366168-1 {}] regionserver.HStore(327): Store=0d1e33fe1a366293d5e32ce198366168/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:25:10,660 INFO [StoreOpener-0d1e33fe1a366293d5e32ce198366168-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:10,660 INFO [StoreOpener-0d1e33fe1a366293d5e32ce198366168-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d1e33fe1a366293d5e32ce198366168 columnFamilyName B 2024-11-08T00:25:10,660 DEBUG [StoreOpener-0d1e33fe1a366293d5e32ce198366168-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:10,661 INFO [StoreOpener-0d1e33fe1a366293d5e32ce198366168-1 {}] regionserver.HStore(327): Store=0d1e33fe1a366293d5e32ce198366168/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:25:10,661 INFO [StoreOpener-0d1e33fe1a366293d5e32ce198366168-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:10,662 INFO [StoreOpener-0d1e33fe1a366293d5e32ce198366168-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d1e33fe1a366293d5e32ce198366168 columnFamilyName C 2024-11-08T00:25:10,662 DEBUG [StoreOpener-0d1e33fe1a366293d5e32ce198366168-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:10,662 INFO [StoreOpener-0d1e33fe1a366293d5e32ce198366168-1 {}] regionserver.HStore(327): Store=0d1e33fe1a366293d5e32ce198366168/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:25:10,662 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1038): replaying wal for 0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:10,663 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:10,664 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:10,665 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1048): stopping wal replay for 0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:10,665 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1060): Cleaning up temporary data for 0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:10,665 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-08T00:25:10,666 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1093): writing seq id for 0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:10,667 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1114): Opened 0d1e33fe1a366293d5e32ce198366168; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70987808, jitterRate=0.0578007698059082}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-08T00:25:10,667 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:10,668 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1006): Region open journal for 0d1e33fe1a366293d5e32ce198366168: Running coprocessor pre-open hook at 1731025510656Writing region info on filesystem at 1731025510656Initializing all the Stores at 1731025510657 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731025510657Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731025510657Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731025510657Cleaning up temporary data from old regions at 1731025510665 (+8 ms)Running coprocessor post-open hooks at 1731025510667 (+2 ms)Region opened successfully at 1731025510667 2024-11-08T00:25:10,668 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., pid=92, masterSystemTime=1731025510652 2024-11-08T00:25:10,670 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:10,670 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:10,670 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=0d1e33fe1a366293d5e32ce198366168, regionState=OPEN, openSeqNum=5, regionLocation=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:10,671 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 because future has completed 2024-11-08T00:25:10,673 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=90 2024-11-08T00:25:10,673 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=90, state=SUCCESS, hasLock=false; OpenRegionProcedure 0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 in 172 msec 2024-11-08T00:25:10,675 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-08T00:25:10,675 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0d1e33fe1a366293d5e32ce198366168, REOPEN/MOVE in 505 msec 2024-11-08T00:25:10,677 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-08T00:25:10,677 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 510 msec 2024-11-08T00:25:10,680 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 537 msec 2024-11-08T00:25:10,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-08T00:25:10,681 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:25:10,681 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ead1e03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:10,682 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:25:10,682 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:25:10,683 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:25:10,683 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:25:10,683 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:25:10,683 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11820010, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:10,683 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:25:10,683 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:25:10,684 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:10,684 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43398, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:25:10,685 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@106b6624, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:10,686 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@515a2061, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:10,686 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:25:10,686 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:25:10,686 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:25:10,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:25:10,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:25:10,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c1d4cb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:10,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:25:10,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:25:10,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:10,688 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43404, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:25:10,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6885233f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:10,692 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15faf055, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:10,692 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:25:10,692 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:25:10,693 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:25:10,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:25:10,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:25:10,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bd11183, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:10,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:25:10,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:25:10,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:10,695 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43414, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:25:10,696 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bfdd648, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:10,696 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30ecdff5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:10,696 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:25:10,696 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:25:10,697 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:25:10,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:25:10,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:25:10,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7eca714b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:10,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:25:10,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:25:10,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:10,699 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43428, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:25:10,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7447e542, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:10,700 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b7b6c31, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:10,700 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:25:10,701 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:25:10,702 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:25:10,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:25:10,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:25:10,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a671340, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:10,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:25:10,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:25:10,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:10,703 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43456, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:25:10,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3db0414d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:10,704 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27da8a0e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:10,705 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:25:10,705 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:25:10,705 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:25:10,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:25:10,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:25:10,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e82bf5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:10,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:25:10,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:25:10,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:10,707 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43468, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:25:10,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22623915, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:10,712 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27e4689b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:10,712 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:25:10,712 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:25:10,713 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:25:10,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:25:10,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:25:10,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ac09d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:10,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:25:10,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:25:10,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:10,714 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43488, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:25:10,715 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3019e256, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:10,716 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@460b6145, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:10,716 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:25:10,716 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:25:10,717 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:25:10,717 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:25:10,717 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:25:10,718 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71529c81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:10,718 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:25:10,718 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:25:10,718 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:10,719 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43508, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:25:10,719 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@907f9bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:10,720 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ebf71a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:10,720 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:25:10,720 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:25:10,721 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:25:10,721 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:25:10,721 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:25:10,721 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17db6e9e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:10,721 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:25:10,722 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:25:10,722 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:10,723 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43512, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:25:10,723 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dedf4e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:10,727 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74c55bd9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:10,727 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:25:10,727 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:25:10,728 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:25:10,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:25:10,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:25:10,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@220c5556, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:10,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:25:10,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:25:10,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:10,730 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43540, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:25:10,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f76b357, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:10,733 DEBUG [Thread-1685 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:25:10,733 DEBUG [Thread-1670 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:25:10,733 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-08T00:25:10,734 DEBUG [Thread-1683 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:25:10,734 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:25:10,734 DEBUG [Thread-1678 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:25:10,734 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:25:10,734 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:25:10,734 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:25:10,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-11-08T00:25:10,735 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47200, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:25:10,736 DEBUG [Thread-1687 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:25:10,736 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:25:10,736 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47216, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:25:10,736 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:25:10,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-08T00:25:10,738 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5] 2024-11-08T00:25:10,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:25:10,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:25:10,738 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:25:10,739 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:25:10,739 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T00:25:10,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5] 2024-11-08T00:25:10,740 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47232, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:25:10,740 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47238, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:25:10,740 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47242, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:25:10,740 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T00:25:10,740 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T00:25:10,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5] 2024-11-08T00:25:10,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5] 2024-11-08T00:25:10,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5] 2024-11-08T00:25:10,749 DEBUG [Thread-1674 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:25:10,750 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:25:10,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:25:10,751 DEBUG [Thread-1676 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:25:10,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:25:10,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:25:10,752 DEBUG [Thread-1689 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:25:10,753 DEBUG [Thread-1689 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:25:10,753 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47256, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:25:10,754 DEBUG [Thread-1689 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:25:10,754 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47250, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:25:10,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5] 2024-11-08T00:25:10,756 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47258, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:25:10,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5] 2024-11-08T00:25:10,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5] 2024-11-08T00:25:10,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:10,760 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:25:10,760 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0d1e33fe1a366293d5e32ce198366168 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T00:25:10,761 DEBUG [Thread-1672 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:25:10,761 DEBUG [Thread-1681 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:25:10,762 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:25:10,762 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:25:10,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:25:10,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:25:10,763 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47272, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:25:10,764 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47286, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:25:10,766 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5] 2024-11-08T00:25:10,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5] 2024-11-08T00:25:10,795 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411088f1a9c69bf8a43f9bc58678632e8e3ca_0d1e33fe1a366293d5e32ce198366168 is 50, key is test_row_0/A:col10/1731025510759/Put/seqid=0 2024-11-08T00:25:10,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742195_1371 (size=14594) 2024-11-08T00:25:10,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:10,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025570816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:10,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:10,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:10,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:10,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:10,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025570815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:10,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:10,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:10,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:10,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:10,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025570818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:10,827 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:10,827 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:10,827 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:10,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:10,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1731025570818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:10,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:10,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:10,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:10,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:10,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025570826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:10,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:10,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:10,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:10,828 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:10,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-08T00:25:10,852 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411088f1a9c69bf8a43f9bc58678632e8e3ca_0d1e33fe1a366293d5e32ce198366168 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411088f1a9c69bf8a43f9bc58678632e8e3ca_0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:10,852 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/3ade3e156ded429a8d3306068b0912fa, store: [table=TestAcidGuarantees family=A region=0d1e33fe1a366293d5e32ce198366168] 2024-11-08T00:25:10,853 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/3ade3e156ded429a8d3306068b0912fa is 175, key is test_row_0/A:col10/1731025510759/Put/seqid=0 2024-11-08T00:25:10,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742196_1372 (size=39549) 2024-11-08T00:25:10,872 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/3ade3e156ded429a8d3306068b0912fa 2024-11-08T00:25:10,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-08T00:25:10,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:10,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. as already flushing 2024-11-08T00:25:10,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:10,894 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:10,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:10,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:10,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/d7c75a50452244d3bd4afafd593b01f7 is 50, key is test_row_0/B:col10/1731025510759/Put/seqid=0 2024-11-08T00:25:10,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742197_1373 (size=12001) 2024-11-08T00:25:10,934 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/d7c75a50452244d3bd4afafd593b01f7 2024-11-08T00:25:10,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:10,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025570937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:10,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:10,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:10,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:10,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:10,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025570938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:10,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:10,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:10,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:10,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:10,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025570938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:10,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:10,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:10,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:10,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:10,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025570938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:10,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:10,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:10,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:10,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:10,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1731025570938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:10,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:10,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:10,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:10,966 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/09198b3429364946bc72ce17f59b79cb is 50, key is test_row_0/C:col10/1731025510759/Put/seqid=0 2024-11-08T00:25:11,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742198_1374 (size=12001) 2024-11-08T00:25:11,016 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/09198b3429364946bc72ce17f59b79cb 2024-11-08T00:25:11,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/3ade3e156ded429a8d3306068b0912fa as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/3ade3e156ded429a8d3306068b0912fa 2024-11-08T00:25:11,040 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/3ade3e156ded429a8d3306068b0912fa, entries=200, sequenceid=16, filesize=38.6 K 2024-11-08T00:25:11,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/d7c75a50452244d3bd4afafd593b01f7 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/d7c75a50452244d3bd4afafd593b01f7 2024-11-08T00:25:11,045 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/d7c75a50452244d3bd4afafd593b01f7, entries=150, sequenceid=16, filesize=11.7 K 2024-11-08T00:25:11,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/09198b3429364946bc72ce17f59b79cb as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/09198b3429364946bc72ce17f59b79cb 2024-11-08T00:25:11,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-08T00:25:11,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:11,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. as already flushing 2024-11-08T00:25:11,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:11,047 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:11,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:11,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:11,056 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/09198b3429364946bc72ce17f59b79cb, entries=150, sequenceid=16, filesize=11.7 K 2024-11-08T00:25:11,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-08T00:25:11,058 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 0d1e33fe1a366293d5e32ce198366168 in 298ms, sequenceid=16, compaction requested=false 2024-11-08T00:25:11,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0d1e33fe1a366293d5e32ce198366168: 2024-11-08T00:25:11,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:11,176 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:25:11,176 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0d1e33fe1a366293d5e32ce198366168 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-08T00:25:11,196 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411081046c4dd056d4c4f8916ebed4f5d1845_0d1e33fe1a366293d5e32ce198366168 is 50, key is test_row_0/A:col10/1731025511168/Put/seqid=0 2024-11-08T00:25:11,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-08T00:25:11,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:11,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. as already flushing 2024-11-08T00:25:11,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:11,201 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:11,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:11,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:11,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742199_1375 (size=14594) 2024-11-08T00:25:11,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:11,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025571209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:11,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:11,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:11,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:11,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:11,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1731025571217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:11,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:11,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:11,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:11,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:11,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025571228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:11,248 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:11,248 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:11,248 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:11,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:11,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025571230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:11,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:11,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:11,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:11,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:11,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025571232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:11,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:11,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:11,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:11,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:11,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025571339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:11,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:11,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:11,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:11,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1731025571339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:11,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:11,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:11,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:11,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:11,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-08T00:25:11,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:11,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. as already flushing 2024-11-08T00:25:11,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:11,357 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:11,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:11,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:11,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:11,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025571357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:11,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:11,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:11,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:11,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:11,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025571357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:11,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:11,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:11,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:11,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:11,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025571359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:11,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:11,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:11,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:11,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-08T00:25:11,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-08T00:25:11,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:11,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. as already flushing 2024-11-08T00:25:11,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:11,511 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:11,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:11,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:11,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:11,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1731025571557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:11,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:11,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:11,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:11,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:11,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025571557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:11,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:11,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:11,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:11,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:11,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025571567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:11,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:11,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:11,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:11,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:11,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025571567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:11,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:11,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:11,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:11,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:11,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025571568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:11,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:11,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:11,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:11,632 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:11,636 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411081046c4dd056d4c4f8916ebed4f5d1845_0d1e33fe1a366293d5e32ce198366168 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411081046c4dd056d4c4f8916ebed4f5d1845_0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:11,637 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/9001e2bc424d493399048645ccc55b6c, store: [table=TestAcidGuarantees family=A region=0d1e33fe1a366293d5e32ce198366168] 2024-11-08T00:25:11,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/9001e2bc424d493399048645ccc55b6c is 175, key is test_row_0/A:col10/1731025511168/Put/seqid=0 2024-11-08T00:25:11,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-08T00:25:11,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:11,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. as already flushing 2024-11-08T00:25:11,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:11,664 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:11,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:11,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:11,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742200_1376 (size=39549) 2024-11-08T00:25:11,689 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/9001e2bc424d493399048645ccc55b6c 2024-11-08T00:25:11,728 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/03be9bcc09d14e96957aacda5a946263 is 50, key is test_row_0/B:col10/1731025511168/Put/seqid=0 2024-11-08T00:25:11,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742201_1377 (size=12001) 2024-11-08T00:25:11,737 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/03be9bcc09d14e96957aacda5a946263 2024-11-08T00:25:11,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/e7eb2546db85401da92df02f008a35fd is 50, key is test_row_0/C:col10/1731025511168/Put/seqid=0 2024-11-08T00:25:11,826 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-08T00:25:11,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:11,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. as already flushing 2024-11-08T00:25:11,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:11,827 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:11,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:11,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:11,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742202_1378 (size=12001) 2024-11-08T00:25:11,832 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/e7eb2546db85401da92df02f008a35fd 2024-11-08T00:25:11,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/9001e2bc424d493399048645ccc55b6c as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/9001e2bc424d493399048645ccc55b6c 2024-11-08T00:25:11,861 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/9001e2bc424d493399048645ccc55b6c, entries=200, sequenceid=41, filesize=38.6 K 2024-11-08T00:25:11,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/03be9bcc09d14e96957aacda5a946263 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/03be9bcc09d14e96957aacda5a946263 2024-11-08T00:25:11,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:11,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1731025571867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:11,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:11,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:11,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:11,877 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/03be9bcc09d14e96957aacda5a946263, entries=150, sequenceid=41, filesize=11.7 K 2024-11-08T00:25:11,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-08T00:25:11,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:11,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025571876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:11,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:11,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:11,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:11,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:11,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025571887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:11,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:11,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025571887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:11,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:11,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:11,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:11,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:11,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:11,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:11,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/e7eb2546db85401da92df02f008a35fd as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/e7eb2546db85401da92df02f008a35fd 2024-11-08T00:25:11,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:11,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025571887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:11,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:11,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:11,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:11,937 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/e7eb2546db85401da92df02f008a35fd, entries=150, sequenceid=41, filesize=11.7 K 2024-11-08T00:25:11,939 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 0d1e33fe1a366293d5e32ce198366168 in 763ms, sequenceid=41, compaction requested=false 2024-11-08T00:25:11,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0d1e33fe1a366293d5e32ce198366168: 2024-11-08T00:25:11,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-08T00:25:11,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:11,994 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing 0d1e33fe1a366293d5e32ce198366168 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T00:25:12,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411087d4d8fce9e794c68b174f844042b57e6_0d1e33fe1a366293d5e32ce198366168 is 50, key is test_row_0/A:col10/1731025511228/Put/seqid=0 2024-11-08T00:25:12,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742203_1379 (size=12154) 2024-11-08T00:25:12,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. as already flushing 2024-11-08T00:25:12,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:12,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:12,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025572457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:12,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:12,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:12,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:12,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025572458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:12,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:12,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:12,476 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:12,476 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:12,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1731025572468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:12,476 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:12,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:12,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:12,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:12,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:12,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025572470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:12,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:12,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:12,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:12,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:12,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:12,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025572475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:12,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:12,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:12,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:12,509 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411087d4d8fce9e794c68b174f844042b57e6_0d1e33fe1a366293d5e32ce198366168 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411087d4d8fce9e794c68b174f844042b57e6_0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:12,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/abded7b318194d89a9e4ff8268fa1d1f, store: [table=TestAcidGuarantees family=A region=0d1e33fe1a366293d5e32ce198366168] 2024-11-08T00:25:12,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/abded7b318194d89a9e4ff8268fa1d1f is 175, key is test_row_0/A:col10/1731025511228/Put/seqid=0 2024-11-08T00:25:12,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742204_1380 (size=30955) 2024-11-08T00:25:12,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:12,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025572588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:12,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:12,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1731025572588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:12,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:12,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:12,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025572588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:12,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:12,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:12,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:12,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:12,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:12,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:12,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:12,596 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:12,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:12,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025572588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:12,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:12,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:12,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:12,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:12,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025572597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:12,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:12,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:12,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:12,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:12,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1731025572807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:12,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:12,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:12,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025572808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:12,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:12,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:12,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:12,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:12,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:12,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025572808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:12,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:12,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:12,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:12,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:12,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:12,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025572808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:12,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:12,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025572808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:12,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:12,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:12,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:12,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:12,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:12,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:12,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-08T00:25:12,919 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/abded7b318194d89a9e4ff8268fa1d1f 2024-11-08T00:25:12,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/abcb5f77e86c4eb6a1a831c79dbec325 is 50, key is test_row_0/B:col10/1731025511228/Put/seqid=0 2024-11-08T00:25:12,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742205_1381 (size=12001) 2024-11-08T00:25:12,949 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/abcb5f77e86c4eb6a1a831c79dbec325 2024-11-08T00:25:12,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/c706c778e9914f1c91ee3d055aa2960d is 50, key is test_row_0/C:col10/1731025511228/Put/seqid=0 2024-11-08T00:25:12,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742206_1382 (size=12001) 2024-11-08T00:25:13,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:13,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025573133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:13,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:13,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1731025573133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:13,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:13,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:13,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:13,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:13,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025573133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:13,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:13,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:13,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:13,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:13,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025573133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:13,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:13,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:13,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:13,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:13,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:13,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:13,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:13,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025573133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:13,161 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:13,161 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:13,161 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:13,392 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/c706c778e9914f1c91ee3d055aa2960d 2024-11-08T00:25:13,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/abded7b318194d89a9e4ff8268fa1d1f as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/abded7b318194d89a9e4ff8268fa1d1f 2024-11-08T00:25:13,408 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/abded7b318194d89a9e4ff8268fa1d1f, entries=150, sequenceid=52, filesize=30.2 K 2024-11-08T00:25:13,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/abcb5f77e86c4eb6a1a831c79dbec325 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/abcb5f77e86c4eb6a1a831c79dbec325 2024-11-08T00:25:13,413 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/abcb5f77e86c4eb6a1a831c79dbec325, entries=150, sequenceid=52, filesize=11.7 K 2024-11-08T00:25:13,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/c706c778e9914f1c91ee3d055aa2960d as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/c706c778e9914f1c91ee3d055aa2960d 2024-11-08T00:25:13,421 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/c706c778e9914f1c91ee3d055aa2960d, entries=150, sequenceid=52, filesize=11.7 K 2024-11-08T00:25:13,424 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 0d1e33fe1a366293d5e32ce198366168 in 1429ms, sequenceid=52, compaction requested=true 2024-11-08T00:25:13,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for 0d1e33fe1a366293d5e32ce198366168: 2024-11-08T00:25:13,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:13,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-11-08T00:25:13,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-11-08T00:25:13,433 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-08T00:25:13,433 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6910 sec 2024-11-08T00:25:13,435 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 2.7010 sec 2024-11-08T00:25:13,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:13,677 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:25:13,677 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0d1e33fe1a366293d5e32ce198366168 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-08T00:25:13,703 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108428fa3b3477c4118ba783d58e33f3f12_0d1e33fe1a366293d5e32ce198366168 is 50, key is test_row_0/A:col10/1731025512438/Put/seqid=0 2024-11-08T00:25:13,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:13,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025573688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:13,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:13,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:13,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:13,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:13,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1731025573690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:13,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:13,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:13,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:13,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:13,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025573699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:13,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:13,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:13,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025573699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:13,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:13,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:13,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:13,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025573700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:13,712 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:13,712 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:13,712 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:13,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:13,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:13,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:13,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742207_1383 (size=14594) 2024-11-08T00:25:13,734 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:13,738 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108428fa3b3477c4118ba783d58e33f3f12_0d1e33fe1a366293d5e32ce198366168 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108428fa3b3477c4118ba783d58e33f3f12_0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:13,745 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/b9a34facbd464d91bebd049fa5c8fc17, store: [table=TestAcidGuarantees family=A region=0d1e33fe1a366293d5e32ce198366168] 2024-11-08T00:25:13,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/b9a34facbd464d91bebd049fa5c8fc17 is 175, key is test_row_0/A:col10/1731025512438/Put/seqid=0 2024-11-08T00:25:13,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742208_1384 (size=39549) 2024-11-08T00:25:13,765 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/b9a34facbd464d91bebd049fa5c8fc17 2024-11-08T00:25:13,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/e35e53b5e42040a4aa695113b7a3a505 is 50, key is test_row_0/B:col10/1731025512438/Put/seqid=0 2024-11-08T00:25:13,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:13,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025573808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:13,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:13,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:13,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:13,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:13,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1731025573809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:13,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:13,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:13,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:13,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742209_1385 (size=12001) 2024-11-08T00:25:13,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:13,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025573817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:13,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:13,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025573817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:13,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:13,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:13,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:13,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:13,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025573817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:13,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:13,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:13,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:13,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:13,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:13,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:14,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:14,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025574018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:14,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:14,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:14,022 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:14,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:14,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1731025574027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:14,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:14,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:14,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:14,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:14,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025574038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:14,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:14,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025574038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:14,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:14,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:14,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:14,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:14,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:14,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:14,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:14,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025574038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:14,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:14,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:14,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:14,113 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T00:25:14,227 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/e35e53b5e42040a4aa695113b7a3a505 2024-11-08T00:25:14,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/4166d9ffae7b4120b463c6dbe3369cb2 is 50, key is test_row_0/C:col10/1731025512438/Put/seqid=0 2024-11-08T00:25:14,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742210_1386 (size=12001) 2024-11-08T00:25:14,274 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/4166d9ffae7b4120b463c6dbe3369cb2 2024-11-08T00:25:14,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/b9a34facbd464d91bebd049fa5c8fc17 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/b9a34facbd464d91bebd049fa5c8fc17 2024-11-08T00:25:14,288 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/b9a34facbd464d91bebd049fa5c8fc17, entries=200, sequenceid=78, filesize=38.6 K 2024-11-08T00:25:14,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/e35e53b5e42040a4aa695113b7a3a505 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/e35e53b5e42040a4aa695113b7a3a505 2024-11-08T00:25:14,300 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/e35e53b5e42040a4aa695113b7a3a505, entries=150, sequenceid=78, filesize=11.7 K 2024-11-08T00:25:14,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/4166d9ffae7b4120b463c6dbe3369cb2 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/4166d9ffae7b4120b463c6dbe3369cb2 2024-11-08T00:25:14,309 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/4166d9ffae7b4120b463c6dbe3369cb2, entries=150, sequenceid=78, filesize=11.7 K 2024-11-08T00:25:14,312 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 0d1e33fe1a366293d5e32ce198366168 in 634ms, sequenceid=78, compaction requested=true 2024-11-08T00:25:14,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0d1e33fe1a366293d5e32ce198366168: 2024-11-08T00:25:14,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d1e33fe1a366293d5e32ce198366168:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T00:25:14,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:25:14,312 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T00:25:14,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d1e33fe1a366293d5e32ce198366168:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T00:25:14,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:25:14,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d1e33fe1a366293d5e32ce198366168:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:25:14,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-08T00:25:14,312 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T00:25:14,313 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 149602 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T00:25:14,313 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): 0d1e33fe1a366293d5e32ce198366168/A is initiating minor compaction (all files) 2024-11-08T00:25:14,314 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d1e33fe1a366293d5e32ce198366168/A in TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:14,314 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/3ade3e156ded429a8d3306068b0912fa, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/9001e2bc424d493399048645ccc55b6c, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/abded7b318194d89a9e4ff8268fa1d1f, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/b9a34facbd464d91bebd049fa5c8fc17] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp, totalSize=146.1 K 2024-11-08T00:25:14,314 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:14,314 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. files: [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/3ade3e156ded429a8d3306068b0912fa, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/9001e2bc424d493399048645ccc55b6c, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/abded7b318194d89a9e4ff8268fa1d1f, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/b9a34facbd464d91bebd049fa5c8fc17] 2024-11-08T00:25:14,314 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T00:25:14,314 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): 0d1e33fe1a366293d5e32ce198366168/B is initiating minor compaction (all files) 2024-11-08T00:25:14,314 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d1e33fe1a366293d5e32ce198366168/B in TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:14,314 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/d7c75a50452244d3bd4afafd593b01f7, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/03be9bcc09d14e96957aacda5a946263, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/abcb5f77e86c4eb6a1a831c79dbec325, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/e35e53b5e42040a4aa695113b7a3a505] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp, totalSize=46.9 K 2024-11-08T00:25:14,315 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ade3e156ded429a8d3306068b0912fa, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731025510749 2024-11-08T00:25:14,315 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting d7c75a50452244d3bd4afafd593b01f7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731025510749 2024-11-08T00:25:14,318 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9001e2bc424d493399048645ccc55b6c, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731025510810 2024-11-08T00:25:14,318 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 03be9bcc09d14e96957aacda5a946263, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731025510810 2024-11-08T00:25:14,318 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting abcb5f77e86c4eb6a1a831c79dbec325, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731025511184 2024-11-08T00:25:14,318 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting abded7b318194d89a9e4ff8268fa1d1f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731025511184 2024-11-08T00:25:14,318 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting e35e53b5e42040a4aa695113b7a3a505, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731025512438 2024-11-08T00:25:14,318 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9a34facbd464d91bebd049fa5c8fc17, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731025512438 2024-11-08T00:25:14,338 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:25:14,338 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0d1e33fe1a366293d5e32ce198366168 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-08T00:25:14,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:14,340 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d1e33fe1a366293d5e32ce198366168#B#compaction#332 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:25:14,342 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0d1e33fe1a366293d5e32ce198366168] 2024-11-08T00:25:14,342 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/ac6303b547894d1583a0bae39b6d0341 is 50, key is test_row_0/B:col10/1731025512438/Put/seqid=0 2024-11-08T00:25:14,352 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411080123d27ad3d6455b88d9d15f2b324fde_0d1e33fe1a366293d5e32ce198366168 store=[table=TestAcidGuarantees family=A region=0d1e33fe1a366293d5e32ce198366168] 2024-11-08T00:25:14,354 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411080123d27ad3d6455b88d9d15f2b324fde_0d1e33fe1a366293d5e32ce198366168, store=[table=TestAcidGuarantees family=A region=0d1e33fe1a366293d5e32ce198366168] 2024-11-08T00:25:14,355 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411080123d27ad3d6455b88d9d15f2b324fde_0d1e33fe1a366293d5e32ce198366168 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0d1e33fe1a366293d5e32ce198366168] 2024-11-08T00:25:14,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110814b82a4967ea4eafa828376776e17f48_0d1e33fe1a366293d5e32ce198366168 is 50, key is test_row_0/A:col10/1731025513697/Put/seqid=0 2024-11-08T00:25:14,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742211_1387 (size=12139) 2024-11-08T00:25:14,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742212_1388 (size=4469) 2024-11-08T00:25:14,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742213_1389 (size=14594) 2024-11-08T00:25:14,405 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:14,410 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110814b82a4967ea4eafa828376776e17f48_0d1e33fe1a366293d5e32ce198366168 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110814b82a4967ea4eafa828376776e17f48_0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:14,411 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d1e33fe1a366293d5e32ce198366168#A#compaction#333 average throughput is 0.35 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:25:14,412 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/947c6958e4ca4dec97be3f4961bdc951 is 175, key is test_row_0/A:col10/1731025512438/Put/seqid=0 2024-11-08T00:25:14,413 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/8237934a842645a1a175adabf738c223, store: [table=TestAcidGuarantees family=A region=0d1e33fe1a366293d5e32ce198366168] 2024-11-08T00:25:14,413 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/8237934a842645a1a175adabf738c223 is 175, key is test_row_0/A:col10/1731025513697/Put/seqid=0 2024-11-08T00:25:14,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742214_1390 (size=39549) 2024-11-08T00:25:14,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742215_1391 (size=31093) 2024-11-08T00:25:14,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:14,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1731025574420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:14,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:14,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:14,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:14,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:14,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025574420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:14,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:14,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:14,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:14,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:14,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025574420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:14,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:14,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:14,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:14,442 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/947c6958e4ca4dec97be3f4961bdc951 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/947c6958e4ca4dec97be3f4961bdc951 2024-11-08T00:25:14,447 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0d1e33fe1a366293d5e32ce198366168/A of 0d1e33fe1a366293d5e32ce198366168 into 947c6958e4ca4dec97be3f4961bdc951(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:25:14,447 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d1e33fe1a366293d5e32ce198366168: 2024-11-08T00:25:14,447 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., storeName=0d1e33fe1a366293d5e32ce198366168/A, priority=12, startTime=1731025514312; duration=0sec 2024-11-08T00:25:14,447 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:25:14,447 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d1e33fe1a366293d5e32ce198366168:A 2024-11-08T00:25:14,447 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T00:25:14,450 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T00:25:14,450 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): 0d1e33fe1a366293d5e32ce198366168/C is initiating minor compaction (all files) 2024-11-08T00:25:14,450 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d1e33fe1a366293d5e32ce198366168/C in TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:14,450 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/09198b3429364946bc72ce17f59b79cb, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/e7eb2546db85401da92df02f008a35fd, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/c706c778e9914f1c91ee3d055aa2960d, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/4166d9ffae7b4120b463c6dbe3369cb2] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp, totalSize=46.9 K 2024-11-08T00:25:14,451 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09198b3429364946bc72ce17f59b79cb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731025510749 2024-11-08T00:25:14,452 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7eb2546db85401da92df02f008a35fd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731025510810 2024-11-08T00:25:14,452 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting c706c778e9914f1c91ee3d055aa2960d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731025511184 2024-11-08T00:25:14,453 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4166d9ffae7b4120b463c6dbe3369cb2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731025512438 2024-11-08T00:25:14,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:14,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:14,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025574439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:14,462 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d1e33fe1a366293d5e32ce198366168#C#compaction#335 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:25:14,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:14,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:14,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:14,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025574441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:14,463 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/704b0fe277bc4b41bb09c5fdddc8946b is 50, key is test_row_0/C:col10/1731025512438/Put/seqid=0 2024-11-08T00:25:14,463 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:14,463 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:14,463 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:14,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742216_1392 (size=12139) 2024-11-08T00:25:14,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:14,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025574548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:14,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:14,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:14,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:14,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:14,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1731025574548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:14,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:14,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025574548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:14,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:14,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:14,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:14,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:14,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:14,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:14,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:14,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025574567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:14,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:14,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025574567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:14,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:14,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:14,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:14,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:14,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:14,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:14,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:14,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1731025574768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:14,770 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:14,770 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:14,770 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:14,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:14,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025574768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:14,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:14,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:14,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:14,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025574768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:14,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:14,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:14,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:14,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:14,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:14,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025574777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:14,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:14,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025574777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:14,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:14,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:14,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:14,785 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:14,785 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:14,785 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:14,792 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/ac6303b547894d1583a0bae39b6d0341 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/ac6303b547894d1583a0bae39b6d0341 2024-11-08T00:25:14,798 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0d1e33fe1a366293d5e32ce198366168/B of 0d1e33fe1a366293d5e32ce198366168 into ac6303b547894d1583a0bae39b6d0341(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:25:14,798 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d1e33fe1a366293d5e32ce198366168: 2024-11-08T00:25:14,798 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., storeName=0d1e33fe1a366293d5e32ce198366168/B, priority=12, startTime=1731025514312; duration=0sec 2024-11-08T00:25:14,798 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:25:14,798 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d1e33fe1a366293d5e32ce198366168:B 2024-11-08T00:25:14,819 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=90, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/8237934a842645a1a175adabf738c223 2024-11-08T00:25:14,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/dee6a9ab03ba485eac3eba2eb836a322 is 50, key is test_row_0/B:col10/1731025513697/Put/seqid=0 2024-11-08T00:25:14,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742217_1393 (size=12001) 2024-11-08T00:25:14,882 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/704b0fe277bc4b41bb09c5fdddc8946b as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/704b0fe277bc4b41bb09c5fdddc8946b 2024-11-08T00:25:14,891 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0d1e33fe1a366293d5e32ce198366168/C of 0d1e33fe1a366293d5e32ce198366168 into 704b0fe277bc4b41bb09c5fdddc8946b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:25:14,891 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d1e33fe1a366293d5e32ce198366168: 2024-11-08T00:25:14,891 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., storeName=0d1e33fe1a366293d5e32ce198366168/C, priority=12, startTime=1731025514312; duration=0sec 2024-11-08T00:25:14,891 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:25:14,891 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d1e33fe1a366293d5e32ce198366168:C 2024-11-08T00:25:14,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-08T00:25:14,920 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:25:14,921 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-08T00:25:14,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-08T00:25:14,923 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T00:25:14,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-08T00:25:14,924 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T00:25:14,924 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T00:25:15,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-08T00:25:15,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-08T00:25:15,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:15,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. as already flushing 2024-11-08T00:25:15,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:15,080 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:15,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:15,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:15,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:15,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1731025575078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:15,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:15,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:15,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:15,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:15,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025575078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:15,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:15,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:15,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:15,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:15,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025575078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:15,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:15,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:15,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:15,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:15,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025575097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:15,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:15,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:15,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:15,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:15,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025575097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:15,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:15,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:15,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:15,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-08T00:25:15,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:15,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. as already flushing 2024-11-08T00:25:15,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:15,234 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:15,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:15,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:15,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-08T00:25:15,266 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/dee6a9ab03ba485eac3eba2eb836a322 2024-11-08T00:25:15,284 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/423de780238b4f13ad857802acf3d785 is 50, key is test_row_0/C:col10/1731025513697/Put/seqid=0 2024-11-08T00:25:15,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742218_1394 (size=12001) 2024-11-08T00:25:15,293 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/423de780238b4f13ad857802acf3d785 2024-11-08T00:25:15,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/8237934a842645a1a175adabf738c223 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/8237934a842645a1a175adabf738c223 2024-11-08T00:25:15,319 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/8237934a842645a1a175adabf738c223, entries=200, sequenceid=90, filesize=38.6 K 2024-11-08T00:25:15,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/dee6a9ab03ba485eac3eba2eb836a322 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/dee6a9ab03ba485eac3eba2eb836a322 2024-11-08T00:25:15,332 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/dee6a9ab03ba485eac3eba2eb836a322, entries=150, sequenceid=90, filesize=11.7 K 2024-11-08T00:25:15,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/423de780238b4f13ad857802acf3d785 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/423de780238b4f13ad857802acf3d785 2024-11-08T00:25:15,345 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/423de780238b4f13ad857802acf3d785, entries=150, sequenceid=90, filesize=11.7 K 2024-11-08T00:25:15,346 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 0d1e33fe1a366293d5e32ce198366168 in 1008ms, sequenceid=90, compaction requested=false 2024-11-08T00:25:15,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0d1e33fe1a366293d5e32ce198366168: 2024-11-08T00:25:15,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-08T00:25:15,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:15,388 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 0d1e33fe1a366293d5e32ce198366168 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-08T00:25:15,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108429dfede8b02420aadc7a57b5053bdd9_0d1e33fe1a366293d5e32ce198366168 is 50, key is test_row_0/A:col10/1731025514438/Put/seqid=0 2024-11-08T00:25:15,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742219_1395 (size=12154) 2024-11-08T00:25:15,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-08T00:25:15,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. as already flushing 2024-11-08T00:25:15,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:15,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:15,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025575619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:15,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:15,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:15,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:15,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:15,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025575628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:15,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:15,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:15,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1731025575628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:15,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:15,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:15,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:15,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025575632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:15,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:15,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:15,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:15,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:15,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:15,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:15,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025575632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:15,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:15,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:15,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:15,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:15,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:15,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025575737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:15,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:15,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:15,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:15,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:15,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025575747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:15,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:15,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:15,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:15,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:15,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025575747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:15,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:15,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:15,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:15,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:15,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1731025575747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:15,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:15,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025575747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:15,755 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:15,756 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:15,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:15,756 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:15,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:15,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:15,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:15,824 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108429dfede8b02420aadc7a57b5053bdd9_0d1e33fe1a366293d5e32ce198366168 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108429dfede8b02420aadc7a57b5053bdd9_0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:15,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/3d637170ec014ab58f0cc2b63cd7d93b, store: [table=TestAcidGuarantees family=A region=0d1e33fe1a366293d5e32ce198366168] 2024-11-08T00:25:15,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/3d637170ec014ab58f0cc2b63cd7d93b is 175, key is test_row_0/A:col10/1731025514438/Put/seqid=0 2024-11-08T00:25:15,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742220_1396 (size=30955) 2024-11-08T00:25:15,858 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/3d637170ec014ab58f0cc2b63cd7d93b 2024-11-08T00:25:15,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/4c1e27074a6049098d2dba491f075637 is 50, key is test_row_0/B:col10/1731025514438/Put/seqid=0 2024-11-08T00:25:15,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742221_1397 (size=12001) 2024-11-08T00:25:15,880 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/4c1e27074a6049098d2dba491f075637 2024-11-08T00:25:15,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/4d8e0cb72dcd45c6afe171658e4db83f is 50, key is test_row_0/C:col10/1731025514438/Put/seqid=0 2024-11-08T00:25:15,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742222_1398 (size=12001) 2024-11-08T00:25:15,905 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/4d8e0cb72dcd45c6afe171658e4db83f 2024-11-08T00:25:15,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/3d637170ec014ab58f0cc2b63cd7d93b as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/3d637170ec014ab58f0cc2b63cd7d93b 2024-11-08T00:25:15,919 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/3d637170ec014ab58f0cc2b63cd7d93b, entries=150, sequenceid=117, filesize=30.2 K 2024-11-08T00:25:15,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/4c1e27074a6049098d2dba491f075637 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/4c1e27074a6049098d2dba491f075637 2024-11-08T00:25:15,924 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/4c1e27074a6049098d2dba491f075637, entries=150, sequenceid=117, filesize=11.7 K 2024-11-08T00:25:15,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/4d8e0cb72dcd45c6afe171658e4db83f as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/4d8e0cb72dcd45c6afe171658e4db83f 2024-11-08T00:25:15,933 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/4d8e0cb72dcd45c6afe171658e4db83f, entries=150, sequenceid=117, filesize=11.7 K 2024-11-08T00:25:15,934 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 0d1e33fe1a366293d5e32ce198366168 in 546ms, sequenceid=117, compaction requested=true 2024-11-08T00:25:15,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 0d1e33fe1a366293d5e32ce198366168: 2024-11-08T00:25:15,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:15,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-08T00:25:15,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-08T00:25:15,941 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-08T00:25:15,941 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0120 sec 2024-11-08T00:25:15,943 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.0200 sec 2024-11-08T00:25:15,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:15,960 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:25:15,961 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0d1e33fe1a366293d5e32ce198366168 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-08T00:25:15,987 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108d2934e4655ff4faca1a5180bf00a8647_0d1e33fe1a366293d5e32ce198366168 is 50, key is test_row_0/A:col10/1731025515628/Put/seqid=0 2024-11-08T00:25:15,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742223_1399 (size=17184) 2024-11-08T00:25:15,998 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:16,003 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108d2934e4655ff4faca1a5180bf00a8647_0d1e33fe1a366293d5e32ce198366168 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108d2934e4655ff4faca1a5180bf00a8647_0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:16,004 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/63d1d411b03548ff9b69d79afda708e4, store: [table=TestAcidGuarantees family=A region=0d1e33fe1a366293d5e32ce198366168] 2024-11-08T00:25:16,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/63d1d411b03548ff9b69d79afda708e4 is 175, key is test_row_0/A:col10/1731025515628/Put/seqid=0 2024-11-08T00:25:16,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742224_1400 (size=48289) 2024-11-08T00:25:16,018 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/63d1d411b03548ff9b69d79afda708e4 2024-11-08T00:25:16,042 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/bf95005ada4245d5988bec7323cc253c is 50, key is test_row_0/B:col10/1731025515628/Put/seqid=0 2024-11-08T00:25:16,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:16,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025576033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:16,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:16,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:16,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1731025576035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:16,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:16,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:16,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:16,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025576035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:16,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:16,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025576040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:16,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:16,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:16,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:16,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:16,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:16,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:16,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:16,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025576049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:16,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:16,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:16,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:16,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:16,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:16,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:16,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742225_1401 (size=12151) 2024-11-08T00:25:16,065 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/bf95005ada4245d5988bec7323cc253c 2024-11-08T00:25:16,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-08T00:25:16,068 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:25:16,069 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-08T00:25:16,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-08T00:25:16,071 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T00:25:16,072 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T00:25:16,072 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T00:25:16,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-08T00:25:16,080 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/e75c4c42b8a74539a4934683529e6704 is 50, key is test_row_0/C:col10/1731025515628/Put/seqid=0 2024-11-08T00:25:16,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742226_1402 (size=12151) 2024-11-08T00:25:16,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:16,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:16,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025576157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:16,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025576157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:16,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:16,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025576157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:16,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:16,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:16,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:16,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:16,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:16,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:16,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:16,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:16,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:16,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:16,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1731025576168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:16,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:16,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025576168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:16,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:16,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:16,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:16,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:16,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:16,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:16,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-08T00:25:16,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-08T00:25:16,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:16,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. as already flushing 2024-11-08T00:25:16,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:16,233 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:16,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:16,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:16,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:16,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025576378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:16,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:16,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025576379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:16,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:16,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025576379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:16,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:16,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:16,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:16,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-08T00:25:16,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:16,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:16,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:16,389 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:16,389 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:16,389 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:16,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:16,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. as already flushing 2024-11-08T00:25:16,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:16,390 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:16,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:16,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:16,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-08T00:25:16,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:16,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1731025576390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:16,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:16,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025576391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:16,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:16,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:16,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:16,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:16,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:16,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:16,485 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/e75c4c42b8a74539a4934683529e6704 2024-11-08T00:25:16,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/63d1d411b03548ff9b69d79afda708e4 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/63d1d411b03548ff9b69d79afda708e4 2024-11-08T00:25:16,493 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/63d1d411b03548ff9b69d79afda708e4, entries=250, sequenceid=132, filesize=47.2 K 2024-11-08T00:25:16,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/bf95005ada4245d5988bec7323cc253c as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/bf95005ada4245d5988bec7323cc253c 2024-11-08T00:25:16,501 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/bf95005ada4245d5988bec7323cc253c, entries=150, sequenceid=132, filesize=11.9 K 2024-11-08T00:25:16,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/e75c4c42b8a74539a4934683529e6704 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/e75c4c42b8a74539a4934683529e6704 2024-11-08T00:25:16,507 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/e75c4c42b8a74539a4934683529e6704, entries=150, sequenceid=132, filesize=11.9 K 2024-11-08T00:25:16,508 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 0d1e33fe1a366293d5e32ce198366168 in 548ms, sequenceid=132, compaction requested=true 2024-11-08T00:25:16,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0d1e33fe1a366293d5e32ce198366168: 2024-11-08T00:25:16,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d1e33fe1a366293d5e32ce198366168:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T00:25:16,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:25:16,509 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T00:25:16,509 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T00:25:16,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d1e33fe1a366293d5e32ce198366168:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T00:25:16,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:25:16,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d1e33fe1a366293d5e32ce198366168:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:25:16,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:25:16,510 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 149886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T00:25:16,510 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): 0d1e33fe1a366293d5e32ce198366168/A is initiating minor compaction (all files) 2024-11-08T00:25:16,510 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d1e33fe1a366293d5e32ce198366168/A in TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:16,510 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/947c6958e4ca4dec97be3f4961bdc951, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/8237934a842645a1a175adabf738c223, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/3d637170ec014ab58f0cc2b63cd7d93b, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/63d1d411b03548ff9b69d79afda708e4] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp, totalSize=146.4 K 2024-11-08T00:25:16,510 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:16,510 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. files: [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/947c6958e4ca4dec97be3f4961bdc951, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/8237934a842645a1a175adabf738c223, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/3d637170ec014ab58f0cc2b63cd7d93b, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/63d1d411b03548ff9b69d79afda708e4] 2024-11-08T00:25:16,512 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 947c6958e4ca4dec97be3f4961bdc951, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731025512438 2024-11-08T00:25:16,512 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48292 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T00:25:16,512 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): 0d1e33fe1a366293d5e32ce198366168/B is initiating minor compaction (all files) 2024-11-08T00:25:16,512 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d1e33fe1a366293d5e32ce198366168/B in TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:16,512 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/ac6303b547894d1583a0bae39b6d0341, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/dee6a9ab03ba485eac3eba2eb836a322, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/4c1e27074a6049098d2dba491f075637, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/bf95005ada4245d5988bec7323cc253c] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp, totalSize=47.2 K 2024-11-08T00:25:16,512 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting ac6303b547894d1583a0bae39b6d0341, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731025512438 2024-11-08T00:25:16,513 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting dee6a9ab03ba485eac3eba2eb836a322, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731025513696 2024-11-08T00:25:16,513 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c1e27074a6049098d2dba491f075637, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731025514418 2024-11-08T00:25:16,513 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting bf95005ada4245d5988bec7323cc253c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731025515628 2024-11-08T00:25:16,515 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8237934a842645a1a175adabf738c223, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731025513678 2024-11-08T00:25:16,516 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d637170ec014ab58f0cc2b63cd7d93b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731025514418 2024-11-08T00:25:16,520 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63d1d411b03548ff9b69d79afda708e4, keycount=250, bloomtype=ROW, size=47.2 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731025515617 2024-11-08T00:25:16,539 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d1e33fe1a366293d5e32ce198366168#B#compaction#344 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:25:16,542 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/f2b004c3f1324e808ab771e269a7004d is 50, key is test_row_0/B:col10/1731025515628/Put/seqid=0 2024-11-08T00:25:16,544 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-08T00:25:16,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:16,544 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 0d1e33fe1a366293d5e32ce198366168 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-08T00:25:16,558 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0d1e33fe1a366293d5e32ce198366168] 2024-11-08T00:25:16,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742227_1403 (size=12425) 2024-11-08T00:25:16,597 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/f2b004c3f1324e808ab771e269a7004d as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/f2b004c3f1324e808ab771e269a7004d 2024-11-08T00:25:16,605 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241108cf5ef8c52d3b47189c5598eb74e8c757_0d1e33fe1a366293d5e32ce198366168 store=[table=TestAcidGuarantees family=A region=0d1e33fe1a366293d5e32ce198366168] 2024-11-08T00:25:16,606 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0d1e33fe1a366293d5e32ce198366168/B of 0d1e33fe1a366293d5e32ce198366168 into f2b004c3f1324e808ab771e269a7004d(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:25:16,606 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d1e33fe1a366293d5e32ce198366168: 2024-11-08T00:25:16,606 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., storeName=0d1e33fe1a366293d5e32ce198366168/B, priority=12, startTime=1731025516509; duration=0sec 2024-11-08T00:25:16,606 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:25:16,606 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d1e33fe1a366293d5e32ce198366168:B 2024-11-08T00:25:16,607 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T00:25:16,607 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241108cf5ef8c52d3b47189c5598eb74e8c757_0d1e33fe1a366293d5e32ce198366168, store=[table=TestAcidGuarantees family=A region=0d1e33fe1a366293d5e32ce198366168] 2024-11-08T00:25:16,608 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108cf5ef8c52d3b47189c5598eb74e8c757_0d1e33fe1a366293d5e32ce198366168 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0d1e33fe1a366293d5e32ce198366168] 2024-11-08T00:25:16,615 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48292 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T00:25:16,615 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): 0d1e33fe1a366293d5e32ce198366168/C is initiating minor compaction (all files) 2024-11-08T00:25:16,615 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d1e33fe1a366293d5e32ce198366168/C in TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:16,616 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/704b0fe277bc4b41bb09c5fdddc8946b, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/423de780238b4f13ad857802acf3d785, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/4d8e0cb72dcd45c6afe171658e4db83f, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/e75c4c42b8a74539a4934683529e6704] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp, totalSize=47.2 K 2024-11-08T00:25:16,616 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 704b0fe277bc4b41bb09c5fdddc8946b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731025512438 2024-11-08T00:25:16,616 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 423de780238b4f13ad857802acf3d785, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731025513696 2024-11-08T00:25:16,617 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d8e0cb72dcd45c6afe171658e4db83f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731025514418 2024-11-08T00:25:16,617 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting e75c4c42b8a74539a4934683529e6704, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731025515628 2024-11-08T00:25:16,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110805d399f545424b5b84f12025c176d4b7_0d1e33fe1a366293d5e32ce198366168 is 50, key is test_row_0/A:col10/1731025516036/Put/seqid=0 2024-11-08T00:25:16,642 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d1e33fe1a366293d5e32ce198366168#C#compaction#347 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:25:16,643 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/2b043bb214394c1bb8830f3657581ae9 is 50, key is test_row_0/C:col10/1731025515628/Put/seqid=0 2024-11-08T00:25:16,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742228_1404 (size=4469) 2024-11-08T00:25:16,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742229_1405 (size=12304) 2024-11-08T00:25:16,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742230_1406 (size=12425) 2024-11-08T00:25:16,669 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/2b043bb214394c1bb8830f3657581ae9 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/2b043bb214394c1bb8830f3657581ae9 2024-11-08T00:25:16,675 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0d1e33fe1a366293d5e32ce198366168/C of 0d1e33fe1a366293d5e32ce198366168 into 2b043bb214394c1bb8830f3657581ae9(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:25:16,675 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d1e33fe1a366293d5e32ce198366168: 2024-11-08T00:25:16,675 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., storeName=0d1e33fe1a366293d5e32ce198366168/C, priority=12, startTime=1731025516509; duration=0sec 2024-11-08T00:25:16,675 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:25:16,675 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d1e33fe1a366293d5e32ce198366168:C 2024-11-08T00:25:16,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:16,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. as already flushing 2024-11-08T00:25:16,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-08T00:25:16,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:16,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1731025576723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:16,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:16,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:16,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:16,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025576729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:16,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:16,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:16,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025576734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:16,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:16,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:16,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:16,740 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:16,740 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:16,740 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:16,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:16,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025576737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:16,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:16,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:16,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:16,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:16,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025576739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:16,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:16,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:16,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:16,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:16,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025576847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:16,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:16,854 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:16,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1731025576847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:16,854 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:16,854 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:16,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:16,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025576847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:16,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:16,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:16,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:16,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:16,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:16,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:16,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:16,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025576857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:16,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:16,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:16,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025576858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:16,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:16,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:16,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:16,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:16,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:17,050 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d1e33fe1a366293d5e32ce198366168#A#compaction#345 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:25:17,051 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/0e1b8dfe07264766ad283b8e17eb25cc is 175, key is test_row_0/A:col10/1731025515628/Put/seqid=0 2024-11-08T00:25:17,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:17,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742231_1407 (size=31379) 2024-11-08T00:25:17,060 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110805d399f545424b5b84f12025c176d4b7_0d1e33fe1a366293d5e32ce198366168 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110805d399f545424b5b84f12025c176d4b7_0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:17,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/4e9b46dc3b6941879ba25734fb1bbc2a, store: [table=TestAcidGuarantees family=A region=0d1e33fe1a366293d5e32ce198366168] 2024-11-08T00:25:17,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/4e9b46dc3b6941879ba25734fb1bbc2a is 175, key is test_row_0/A:col10/1731025516036/Put/seqid=0 2024-11-08T00:25:17,064 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/0e1b8dfe07264766ad283b8e17eb25cc as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/0e1b8dfe07264766ad283b8e17eb25cc 2024-11-08T00:25:17,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:17,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025577058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:17,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:17,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1731025577058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:17,065 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:17,065 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:17,065 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:17,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:17,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:17,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:17,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:17,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025577058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:17,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:17,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:17,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:17,074 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0d1e33fe1a366293d5e32ce198366168/A of 0d1e33fe1a366293d5e32ce198366168 into 0e1b8dfe07264766ad283b8e17eb25cc(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:25:17,074 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d1e33fe1a366293d5e32ce198366168: 2024-11-08T00:25:17,074 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., storeName=0d1e33fe1a366293d5e32ce198366168/A, priority=12, startTime=1731025516508; duration=0sec 2024-11-08T00:25:17,074 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:25:17,074 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d1e33fe1a366293d5e32ce198366168:A 2024-11-08T00:25:17,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742232_1408 (size=31105) 2024-11-08T00:25:17,082 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=153, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/4e9b46dc3b6941879ba25734fb1bbc2a 2024-11-08T00:25:17,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:17,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025577077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:17,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:17,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025577077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:17,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:17,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:17,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:17,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:17,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:17,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:17,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/e6f298eacc744444bf3c1c7cf23d1ca9 is 50, key is test_row_0/B:col10/1731025516036/Put/seqid=0 2024-11-08T00:25:17,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742233_1409 (size=12151) 2024-11-08T00:25:17,107 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/e6f298eacc744444bf3c1c7cf23d1ca9 2024-11-08T00:25:17,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/3874f50494bd47df85156830d23c0ace is 50, key is test_row_0/C:col10/1731025516036/Put/seqid=0 2024-11-08T00:25:17,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742234_1410 (size=12151) 2024-11-08T00:25:17,139 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/3874f50494bd47df85156830d23c0ace 2024-11-08T00:25:17,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/4e9b46dc3b6941879ba25734fb1bbc2a as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/4e9b46dc3b6941879ba25734fb1bbc2a 2024-11-08T00:25:17,153 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/4e9b46dc3b6941879ba25734fb1bbc2a, entries=150, sequenceid=153, filesize=30.4 K 2024-11-08T00:25:17,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/e6f298eacc744444bf3c1c7cf23d1ca9 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/e6f298eacc744444bf3c1c7cf23d1ca9 2024-11-08T00:25:17,163 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/e6f298eacc744444bf3c1c7cf23d1ca9, entries=150, sequenceid=153, filesize=11.9 K 2024-11-08T00:25:17,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/3874f50494bd47df85156830d23c0ace as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/3874f50494bd47df85156830d23c0ace 2024-11-08T00:25:17,169 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/3874f50494bd47df85156830d23c0ace, entries=150, sequenceid=153, filesize=11.9 K 2024-11-08T00:25:17,170 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 0d1e33fe1a366293d5e32ce198366168 in 626ms, sequenceid=153, compaction requested=false 2024-11-08T00:25:17,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 0d1e33fe1a366293d5e32ce198366168: 2024-11-08T00:25:17,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:17,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-08T00:25:17,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-08T00:25:17,174 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-08T00:25:17,174 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1000 sec 2024-11-08T00:25:17,176 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.1060 sec 2024-11-08T00:25:17,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-08T00:25:17,217 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:25:17,218 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-08T00:25:17,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-08T00:25:17,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-08T00:25:17,219 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T00:25:17,220 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T00:25:17,220 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T00:25:17,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-08T00:25:17,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-08T00:25:17,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:17,374 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 0d1e33fe1a366293d5e32ce198366168 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-08T00:25:17,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108af4562cf07294a298715048c508ddb0d_0d1e33fe1a366293d5e32ce198366168 is 50, key is test_row_0/A:col10/1731025516737/Put/seqid=0 2024-11-08T00:25:17,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:17,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. as already flushing 2024-11-08T00:25:17,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742235_1411 (size=12304) 2024-11-08T00:25:17,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:17,398 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108af4562cf07294a298715048c508ddb0d_0d1e33fe1a366293d5e32ce198366168 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108af4562cf07294a298715048c508ddb0d_0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:17,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/0745d1a501de4885bf8fe6084a730058, store: [table=TestAcidGuarantees family=A region=0d1e33fe1a366293d5e32ce198366168] 2024-11-08T00:25:17,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/0745d1a501de4885bf8fe6084a730058 is 175, key is test_row_0/A:col10/1731025516737/Put/seqid=0 2024-11-08T00:25:17,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742236_1412 (size=31105) 2024-11-08T00:25:17,409 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=171, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/0745d1a501de4885bf8fe6084a730058 2024-11-08T00:25:17,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/c5c0fe94e90b45f3bc9b2b5c7e0d492b is 50, key is test_row_0/B:col10/1731025516737/Put/seqid=0 2024-11-08T00:25:17,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742237_1413 (size=12151) 2024-11-08T00:25:17,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:17,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025577446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:17,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:17,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025577447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:17,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:17,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1731025577448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:17,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:17,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025577450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:17,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:17,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:17,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:17,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:17,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:17,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:17,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:17,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:17,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:17,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:17,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:17,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:17,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:17,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025577455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:17,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:17,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:17,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:17,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-08T00:25:17,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:17,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025577577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:17,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:17,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:17,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:17,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:17,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025577577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:17,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:17,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:17,584 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:17,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:17,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025577577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:17,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:17,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:17,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:17,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:17,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1731025577577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:17,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:17,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:17,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:17,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:17,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025577577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:17,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:17,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:17,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:17,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:17,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025577787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:17,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:17,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025577789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:17,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:17,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025577789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:17,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:17,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:17,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:17,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:17,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:17,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:17,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:17,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:17,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:17,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:17,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1731025577797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:17,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:17,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:17,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025577800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:17,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:17,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:17,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:17,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:17,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:17,832 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/c5c0fe94e90b45f3bc9b2b5c7e0d492b 2024-11-08T00:25:17,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/1649b56b39954ec8bc22097635f1050c is 50, key is test_row_0/C:col10/1731025516737/Put/seqid=0 2024-11-08T00:25:17,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-08T00:25:17,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742238_1414 (size=12151) 2024-11-08T00:25:18,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:18,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025578107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:18,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:18,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:18,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:18,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:18,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025578107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:18,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:18,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:18,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:18,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:18,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025578107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:18,118 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:18,118 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:18,118 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:18,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:18,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025578117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:18,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:18,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:18,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:18,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:18,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1731025578118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:18,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:18,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:18,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:18,313 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/1649b56b39954ec8bc22097635f1050c 2024-11-08T00:25:18,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/0745d1a501de4885bf8fe6084a730058 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/0745d1a501de4885bf8fe6084a730058 2024-11-08T00:25:18,322 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/0745d1a501de4885bf8fe6084a730058, entries=150, sequenceid=171, filesize=30.4 K 2024-11-08T00:25:18,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/c5c0fe94e90b45f3bc9b2b5c7e0d492b as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/c5c0fe94e90b45f3bc9b2b5c7e0d492b 2024-11-08T00:25:18,328 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/c5c0fe94e90b45f3bc9b2b5c7e0d492b, entries=150, sequenceid=171, filesize=11.9 K 2024-11-08T00:25:18,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/1649b56b39954ec8bc22097635f1050c as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/1649b56b39954ec8bc22097635f1050c 2024-11-08T00:25:18,333 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/1649b56b39954ec8bc22097635f1050c, entries=150, sequenceid=171, filesize=11.9 K 2024-11-08T00:25:18,334 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 0d1e33fe1a366293d5e32ce198366168 in 960ms, sequenceid=171, compaction requested=true 2024-11-08T00:25:18,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 0d1e33fe1a366293d5e32ce198366168: 2024-11-08T00:25:18,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:18,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-08T00:25:18,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-08T00:25:18,338 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-08T00:25:18,338 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1160 sec 2024-11-08T00:25:18,341 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.1210 sec 2024-11-08T00:25:18,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-08T00:25:18,358 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:25:18,359 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-08T00:25:18,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-08T00:25:18,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-08T00:25:18,364 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T00:25:18,365 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T00:25:18,365 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T00:25:18,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-08T00:25:18,519 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-08T00:25:18,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:18,519 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 0d1e33fe1a366293d5e32ce198366168 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-08T00:25:18,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108938d75b3f3184bba9a46b9d175be8042_0d1e33fe1a366293d5e32ce198366168 is 50, key is test_row_0/A:col10/1731025517446/Put/seqid=0 2024-11-08T00:25:18,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742239_1415 (size=12304) 2024-11-08T00:25:18,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:18,574 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108938d75b3f3184bba9a46b9d175be8042_0d1e33fe1a366293d5e32ce198366168 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108938d75b3f3184bba9a46b9d175be8042_0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:18,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/b17bb20c54714d45945105033f843928, store: [table=TestAcidGuarantees family=A region=0d1e33fe1a366293d5e32ce198366168] 2024-11-08T00:25:18,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/b17bb20c54714d45945105033f843928 is 175, key is test_row_0/A:col10/1731025517446/Put/seqid=0 2024-11-08T00:25:18,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742240_1416 (size=31105) 2024-11-08T00:25:18,591 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=192, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/b17bb20c54714d45945105033f843928 2024-11-08T00:25:18,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/f04d8b1d57ec45cd9e071605d501c116 is 50, key is test_row_0/B:col10/1731025517446/Put/seqid=0 2024-11-08T00:25:18,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742241_1417 (size=12151) 2024-11-08T00:25:18,627 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/f04d8b1d57ec45cd9e071605d501c116 2024-11-08T00:25:18,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:18,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. as already flushing 2024-11-08T00:25:18,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/81faea1e3f6c446fb2bf982e92e3f965 is 50, key is test_row_0/C:col10/1731025517446/Put/seqid=0 2024-11-08T00:25:18,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742242_1418 (size=12151) 2024-11-08T00:25:18,657 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/81faea1e3f6c446fb2bf982e92e3f965 2024-11-08T00:25:18,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/b17bb20c54714d45945105033f843928 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/b17bb20c54714d45945105033f843928 2024-11-08T00:25:18,674 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/b17bb20c54714d45945105033f843928, entries=150, sequenceid=192, filesize=30.4 K 2024-11-08T00:25:18,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:18,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-08T00:25:18,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1731025578658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:18,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:18,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:18,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:18,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:18,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025578659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:18,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:18,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025578667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:18,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:18,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:18,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:18,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:18,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:18,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:18,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:18,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025578671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:18,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:18,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:18,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:18,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:18,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025578673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:18,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:18,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:18,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:18,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/f04d8b1d57ec45cd9e071605d501c116 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/f04d8b1d57ec45cd9e071605d501c116 2024-11-08T00:25:18,696 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/f04d8b1d57ec45cd9e071605d501c116, entries=150, sequenceid=192, filesize=11.9 K 2024-11-08T00:25:18,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/81faea1e3f6c446fb2bf982e92e3f965 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/81faea1e3f6c446fb2bf982e92e3f965 2024-11-08T00:25:18,707 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/81faea1e3f6c446fb2bf982e92e3f965, entries=150, sequenceid=192, filesize=11.9 K 2024-11-08T00:25:18,708 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 0d1e33fe1a366293d5e32ce198366168 in 189ms, sequenceid=192, compaction requested=true 2024-11-08T00:25:18,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 0d1e33fe1a366293d5e32ce198366168: 2024-11-08T00:25:18,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:18,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-08T00:25:18,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-08T00:25:18,714 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-08T00:25:18,714 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 347 msec 2024-11-08T00:25:18,716 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 356 msec 2024-11-08T00:25:18,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:18,795 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:25:18,795 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0d1e33fe1a366293d5e32ce198366168 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-08T00:25:18,838 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108cba703e603164a0f90a31f9e5a1778ec_0d1e33fe1a366293d5e32ce198366168 is 50, key is test_row_0/A:col10/1731025518794/Put/seqid=0 2024-11-08T00:25:18,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742243_1419 (size=12304) 2024-11-08T00:25:18,869 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:18,874 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108cba703e603164a0f90a31f9e5a1778ec_0d1e33fe1a366293d5e32ce198366168 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108cba703e603164a0f90a31f9e5a1778ec_0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:18,875 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/4eaffac0042e411e9fb18a4ebd7a50f4, store: [table=TestAcidGuarantees family=A region=0d1e33fe1a366293d5e32ce198366168] 2024-11-08T00:25:18,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/4eaffac0042e411e9fb18a4ebd7a50f4 is 175, key is test_row_0/A:col10/1731025518794/Put/seqid=0 2024-11-08T00:25:18,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:18,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025578862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:18,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:18,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:18,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:18,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:18,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025578864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:18,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:18,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:18,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:18,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:18,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025578873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:18,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:18,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:18,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:18,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:18,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025578879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:18,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:18,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:18,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:18,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:18,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1731025578889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:18,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:18,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:18,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:18,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742244_1420 (size=31105) 2024-11-08T00:25:18,923 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=208, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/4eaffac0042e411e9fb18a4ebd7a50f4 2024-11-08T00:25:18,948 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/00ca2c40dbff43d4bec5d60944e47bf4 is 50, key is test_row_0/B:col10/1731025518794/Put/seqid=0 2024-11-08T00:25:18,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-08T00:25:18,987 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:25:18,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742245_1421 (size=12151) 2024-11-08T00:25:18,988 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-08T00:25:18,989 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/00ca2c40dbff43d4bec5d60944e47bf4 2024-11-08T00:25:18,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-08T00:25:18,993 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T00:25:18,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-08T00:25:18,994 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T00:25:18,994 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T00:25:19,015 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/ed2eb1f4e9c64a63866c4a70b00c52f1 is 50, key is test_row_0/C:col10/1731025518794/Put/seqid=0 2024-11-08T00:25:19,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:19,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025578997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:19,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:19,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:19,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025579000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:19,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:19,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:19,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:19,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025579000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:19,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:19,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:19,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:19,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:19,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:19,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:19,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:19,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025579017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:19,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:19,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:19,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:19,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:19,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1731025579017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:19,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:19,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:19,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:19,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742246_1422 (size=12151) 2024-11-08T00:25:19,070 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/ed2eb1f4e9c64a63866c4a70b00c52f1 2024-11-08T00:25:19,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/4eaffac0042e411e9fb18a4ebd7a50f4 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/4eaffac0042e411e9fb18a4ebd7a50f4 2024-11-08T00:25:19,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/4eaffac0042e411e9fb18a4ebd7a50f4, entries=150, sequenceid=208, filesize=30.4 K 2024-11-08T00:25:19,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/00ca2c40dbff43d4bec5d60944e47bf4 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/00ca2c40dbff43d4bec5d60944e47bf4 2024-11-08T00:25:19,093 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/00ca2c40dbff43d4bec5d60944e47bf4, entries=150, sequenceid=208, filesize=11.9 K 2024-11-08T00:25:19,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/ed2eb1f4e9c64a63866c4a70b00c52f1 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/ed2eb1f4e9c64a63866c4a70b00c52f1 2024-11-08T00:25:19,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-08T00:25:19,097 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/ed2eb1f4e9c64a63866c4a70b00c52f1, entries=150, sequenceid=208, filesize=11.9 K 2024-11-08T00:25:19,098 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 0d1e33fe1a366293d5e32ce198366168 in 303ms, sequenceid=208, compaction requested=true 2024-11-08T00:25:19,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0d1e33fe1a366293d5e32ce198366168: 2024-11-08T00:25:19,098 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-08T00:25:19,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d1e33fe1a366293d5e32ce198366168:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T00:25:19,100 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 155799 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-08T00:25:19,100 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): 0d1e33fe1a366293d5e32ce198366168/A is initiating minor compaction (all files) 2024-11-08T00:25:19,100 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d1e33fe1a366293d5e32ce198366168/A in TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:19,100 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/0e1b8dfe07264766ad283b8e17eb25cc, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/4e9b46dc3b6941879ba25734fb1bbc2a, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/0745d1a501de4885bf8fe6084a730058, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/b17bb20c54714d45945105033f843928, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/4eaffac0042e411e9fb18a4ebd7a50f4] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp, totalSize=152.1 K 2024-11-08T00:25:19,100 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:19,100 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. files: [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/0e1b8dfe07264766ad283b8e17eb25cc, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/4e9b46dc3b6941879ba25734fb1bbc2a, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/0745d1a501de4885bf8fe6084a730058, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/b17bb20c54714d45945105033f843928, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/4eaffac0042e411e9fb18a4ebd7a50f4] 2024-11-08T00:25:19,100 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e1b8dfe07264766ad283b8e17eb25cc, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731025515628 2024-11-08T00:25:19,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:25:19,101 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-08T00:25:19,101 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e9b46dc3b6941879ba25734fb1bbc2a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731025516025 2024-11-08T00:25:19,101 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0745d1a501de4885bf8fe6084a730058, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731025516732 2024-11-08T00:25:19,102 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting b17bb20c54714d45945105033f843928, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731025517446 2024-11-08T00:25:19,102 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4eaffac0042e411e9fb18a4ebd7a50f4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731025518665 2024-11-08T00:25:19,102 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61029 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-08T00:25:19,103 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): 0d1e33fe1a366293d5e32ce198366168/B is initiating minor compaction (all files) 2024-11-08T00:25:19,103 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d1e33fe1a366293d5e32ce198366168/B in TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:19,103 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/f2b004c3f1324e808ab771e269a7004d, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/e6f298eacc744444bf3c1c7cf23d1ca9, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/c5c0fe94e90b45f3bc9b2b5c7e0d492b, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/f04d8b1d57ec45cd9e071605d501c116, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/00ca2c40dbff43d4bec5d60944e47bf4] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp, totalSize=59.6 K 2024-11-08T00:25:19,103 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting f2b004c3f1324e808ab771e269a7004d, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731025515628 2024-11-08T00:25:19,104 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting e6f298eacc744444bf3c1c7cf23d1ca9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731025516025 2024-11-08T00:25:19,104 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting c5c0fe94e90b45f3bc9b2b5c7e0d492b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731025516732 2024-11-08T00:25:19,105 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting f04d8b1d57ec45cd9e071605d501c116, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731025517446 2024-11-08T00:25:19,105 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 00ca2c40dbff43d4bec5d60944e47bf4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731025518665 2024-11-08T00:25:19,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d1e33fe1a366293d5e32ce198366168:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T00:25:19,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:25:19,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d1e33fe1a366293d5e32ce198366168:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:25:19,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:25:19,125 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0d1e33fe1a366293d5e32ce198366168] 2024-11-08T00:25:19,127 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d1e33fe1a366293d5e32ce198366168#B#compaction#359 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:25:19,127 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/2453a3d6405b4208b74fc357e1a04ef2 is 50, key is test_row_0/B:col10/1731025518794/Put/seqid=0 2024-11-08T00:25:19,133 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411089131e6a218114f538ee7f70ca54cb130_0d1e33fe1a366293d5e32ce198366168 store=[table=TestAcidGuarantees family=A region=0d1e33fe1a366293d5e32ce198366168] 2024-11-08T00:25:19,136 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411089131e6a218114f538ee7f70ca54cb130_0d1e33fe1a366293d5e32ce198366168, store=[table=TestAcidGuarantees family=A region=0d1e33fe1a366293d5e32ce198366168] 2024-11-08T00:25:19,136 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411089131e6a218114f538ee7f70ca54cb130_0d1e33fe1a366293d5e32ce198366168 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0d1e33fe1a366293d5e32ce198366168] 2024-11-08T00:25:19,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-08T00:25:19,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:19,153 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 0d1e33fe1a366293d5e32ce198366168 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-08T00:25:19,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742247_1423 (size=12595) 2024-11-08T00:25:19,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742248_1424 (size=4469) 2024-11-08T00:25:19,225 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d1e33fe1a366293d5e32ce198366168#A#compaction#360 average throughput is 0.24 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:25:19,226 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/5bd4292914e24444b07d971cc6d6af0f is 175, key is test_row_0/A:col10/1731025518794/Put/seqid=0 2024-11-08T00:25:19,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. as already flushing 2024-11-08T00:25:19,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:19,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411083c5bf1e608de49fcace955829cbf05cb_0d1e33fe1a366293d5e32ce198366168 is 50, key is test_row_0/A:col10/1731025518887/Put/seqid=0 2024-11-08T00:25:19,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742249_1425 (size=31549) 2024-11-08T00:25:19,305 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/5bd4292914e24444b07d971cc6d6af0f as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/5bd4292914e24444b07d971cc6d6af0f 2024-11-08T00:25:19,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-08T00:25:19,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:19,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025579286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:19,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:19,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1731025579289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:19,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:19,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:19,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:19,316 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 0d1e33fe1a366293d5e32ce198366168/A of 0d1e33fe1a366293d5e32ce198366168 into 5bd4292914e24444b07d971cc6d6af0f(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:25:19,316 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:19,316 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d1e33fe1a366293d5e32ce198366168: 2024-11-08T00:25:19,316 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:19,316 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:19,316 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., storeName=0d1e33fe1a366293d5e32ce198366168/A, priority=11, startTime=1731025519098; duration=0sec 2024-11-08T00:25:19,316 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:25:19,316 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d1e33fe1a366293d5e32ce198366168:A 2024-11-08T00:25:19,316 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-08T00:25:19,318 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61029 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-08T00:25:19,318 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): 0d1e33fe1a366293d5e32ce198366168/C is initiating minor compaction (all files) 2024-11-08T00:25:19,318 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d1e33fe1a366293d5e32ce198366168/C in TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:19,318 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/2b043bb214394c1bb8830f3657581ae9, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/3874f50494bd47df85156830d23c0ace, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/1649b56b39954ec8bc22097635f1050c, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/81faea1e3f6c446fb2bf982e92e3f965, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/ed2eb1f4e9c64a63866c4a70b00c52f1] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp, totalSize=59.6 K 2024-11-08T00:25:19,318 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b043bb214394c1bb8830f3657581ae9, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731025515628 2024-11-08T00:25:19,319 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3874f50494bd47df85156830d23c0ace, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731025516025 2024-11-08T00:25:19,319 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1649b56b39954ec8bc22097635f1050c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731025516732 2024-11-08T00:25:19,319 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81faea1e3f6c446fb2bf982e92e3f965, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731025517446 2024-11-08T00:25:19,319 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed2eb1f4e9c64a63866c4a70b00c52f1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731025518665 2024-11-08T00:25:19,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:19,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025579308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:19,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:19,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:19,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:19,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742250_1426 (size=12304) 2024-11-08T00:25:19,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:19,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025579310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:19,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:19,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:19,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:19,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:19,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025579312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:19,335 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d1e33fe1a366293d5e32ce198366168#C#compaction#362 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:25:19,336 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/a570989d8d124566bec065cd6fe4f9d4 is 50, key is test_row_0/C:col10/1731025518794/Put/seqid=0 2024-11-08T00:25:19,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:19,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:19,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:19,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742251_1427 (size=12595) 2024-11-08T00:25:19,383 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/a570989d8d124566bec065cd6fe4f9d4 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/a570989d8d124566bec065cd6fe4f9d4 2024-11-08T00:25:19,389 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 0d1e33fe1a366293d5e32ce198366168/C of 0d1e33fe1a366293d5e32ce198366168 into a570989d8d124566bec065cd6fe4f9d4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:25:19,389 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d1e33fe1a366293d5e32ce198366168: 2024-11-08T00:25:19,389 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., storeName=0d1e33fe1a366293d5e32ce198366168/C, priority=11, startTime=1731025519106; duration=0sec 2024-11-08T00:25:19,389 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:25:19,389 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d1e33fe1a366293d5e32ce198366168:C 2024-11-08T00:25:19,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:19,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1731025579418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:19,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:19,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:19,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:19,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:19,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025579437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:19,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:19,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:19,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:19,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:19,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025579437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:19,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:19,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:19,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:19,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:19,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025579437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:19,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:19,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:19,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:19,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:19,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025579447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:19,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:19,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:19,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:19,609 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/2453a3d6405b4208b74fc357e1a04ef2 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/2453a3d6405b4208b74fc357e1a04ef2 2024-11-08T00:25:19,613 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 0d1e33fe1a366293d5e32ce198366168/B of 0d1e33fe1a366293d5e32ce198366168 into 2453a3d6405b4208b74fc357e1a04ef2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:25:19,614 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d1e33fe1a366293d5e32ce198366168: 2024-11-08T00:25:19,614 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., storeName=0d1e33fe1a366293d5e32ce198366168/B, priority=11, startTime=1731025519101; duration=0sec 2024-11-08T00:25:19,615 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:25:19,615 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d1e33fe1a366293d5e32ce198366168:B 2024-11-08T00:25:19,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-08T00:25:19,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:19,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1731025579647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:19,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:19,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:19,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:19,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:19,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025579658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:19,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:19,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:19,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:19,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:19,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025579658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:19,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:19,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:19,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:19,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:19,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025579658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:19,678 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:19,678 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:19,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:19,678 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:19,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025579667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:19,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:19,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:19,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:19,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:19,759 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411083c5bf1e608de49fcace955829cbf05cb_0d1e33fe1a366293d5e32ce198366168 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411083c5bf1e608de49fcace955829cbf05cb_0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:19,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/791c5f4f0c4443ef8135a8bf0d633b81, store: [table=TestAcidGuarantees family=A region=0d1e33fe1a366293d5e32ce198366168] 2024-11-08T00:25:19,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/791c5f4f0c4443ef8135a8bf0d633b81 is 175, key is test_row_0/A:col10/1731025518887/Put/seqid=0 2024-11-08T00:25:19,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742252_1428 (size=31105) 2024-11-08T00:25:19,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:19,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1731025579967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:19,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:19,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:19,976 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:19,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:19,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025579977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:19,987 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:19,987 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:19,987 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:19,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:19,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025579988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:19,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:19,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025579988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:19,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:19,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:19,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:20,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:20,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:20,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:20,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:20,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025579988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:20,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:20,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:20,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:20,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-08T00:25:20,195 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=228, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/791c5f4f0c4443ef8135a8bf0d633b81 2024-11-08T00:25:20,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/c1c8e298011a442a858619408a770181 is 50, key is test_row_0/B:col10/1731025518887/Put/seqid=0 2024-11-08T00:25:20,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742253_1429 (size=12151) 2024-11-08T00:25:20,267 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/c1c8e298011a442a858619408a770181 2024-11-08T00:25:20,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/06b10b58ddc4474c950e6fb116dbbc86 is 50, key is test_row_0/C:col10/1731025518887/Put/seqid=0 2024-11-08T00:25:20,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742254_1430 (size=12151) 2024-11-08T00:25:20,302 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/06b10b58ddc4474c950e6fb116dbbc86 2024-11-08T00:25:20,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/791c5f4f0c4443ef8135a8bf0d633b81 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/791c5f4f0c4443ef8135a8bf0d633b81 2024-11-08T00:25:20,313 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/791c5f4f0c4443ef8135a8bf0d633b81, entries=150, sequenceid=228, filesize=30.4 K 2024-11-08T00:25:20,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/c1c8e298011a442a858619408a770181 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/c1c8e298011a442a858619408a770181 2024-11-08T00:25:20,319 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/c1c8e298011a442a858619408a770181, entries=150, sequenceid=228, filesize=11.9 K 2024-11-08T00:25:20,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/06b10b58ddc4474c950e6fb116dbbc86 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/06b10b58ddc4474c950e6fb116dbbc86 2024-11-08T00:25:20,324 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/06b10b58ddc4474c950e6fb116dbbc86, entries=150, sequenceid=228, filesize=11.9 K 2024-11-08T00:25:20,325 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 0d1e33fe1a366293d5e32ce198366168 in 1172ms, sequenceid=228, compaction requested=false 2024-11-08T00:25:20,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 0d1e33fe1a366293d5e32ce198366168: 2024-11-08T00:25:20,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:20,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-08T00:25:20,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-08T00:25:20,329 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-08T00:25:20,329 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3330 sec 2024-11-08T00:25:20,331 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.3420 sec 2024-11-08T00:25:20,502 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:25:20,502 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0d1e33fe1a366293d5e32ce198366168 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-08T00:25:20,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:20,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411087359cd17d9394989b9dcfadf6ab8ebc8_0d1e33fe1a366293d5e32ce198366168 is 50, key is test_row_0/A:col10/1731025519308/Put/seqid=0 2024-11-08T00:25:20,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742255_1431 (size=14794) 2024-11-08T00:25:20,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:20,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1731025580596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:20,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:20,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:20,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:20,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:20,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025580599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:20,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:20,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:20,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:20,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:20,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025580600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:20,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:20,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025580602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:20,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:20,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:20,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:20,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:20,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025580608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:20,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:20,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:20,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:20,638 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:20,638 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:20,638 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:20,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:20,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1731025580727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:20,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:20,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:20,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:20,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:20,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025580738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:20,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:20,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025580738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:20,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:20,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:20,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:20,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:20,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:20,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:20,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:20,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025580747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:20,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:20,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:20,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:20,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:20,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025580748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:20,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:20,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:20,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:20,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:20,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1731025580947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:20,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:20,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:20,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:20,979 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:20,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:20,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025580968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:20,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:20,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:20,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:20,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:20,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025580968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:20,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:20,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:20,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:20,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:20,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025580968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:20,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:20,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:20,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:20,988 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411087359cd17d9394989b9dcfadf6ab8ebc8_0d1e33fe1a366293d5e32ce198366168 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411087359cd17d9394989b9dcfadf6ab8ebc8_0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:20,990 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/1d3a1e30ada54cf785b89fb745d5435a, store: [table=TestAcidGuarantees family=A region=0d1e33fe1a366293d5e32ce198366168] 2024-11-08T00:25:20,991 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/1d3a1e30ada54cf785b89fb745d5435a is 175, key is test_row_0/A:col10/1731025519308/Put/seqid=0 2024-11-08T00:25:20,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:20,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025580976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:20,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:20,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:20,995 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:21,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742256_1432 (size=39749) 2024-11-08T00:25:21,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-08T00:25:21,139 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:25:21,139 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-08T00:25:21,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-08T00:25:21,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-08T00:25:21,144 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T00:25:21,144 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T00:25:21,145 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T00:25:21,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-08T00:25:21,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:21,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1731025581267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:21,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:21,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:21,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:21,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-08T00:25:21,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:21,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. as already flushing 2024-11-08T00:25:21,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:21,298 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:21,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:21,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:21,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025581287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:21,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:21,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:21,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:21,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:21,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:21,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025581288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:21,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:21,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:21,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:21,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:21,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025581288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:21,305 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:21,305 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:21,305 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:21,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:21,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025581308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:21,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:21,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:21,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:21,429 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=248, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/1d3a1e30ada54cf785b89fb745d5435a 2024-11-08T00:25:21,437 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/787d05fcb686498b827264bba6fe9e26 is 50, key is test_row_0/B:col10/1731025519308/Put/seqid=0 2024-11-08T00:25:21,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-08T00:25:21,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:21,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. as already flushing 2024-11-08T00:25:21,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:21,455 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:21,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:21,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:21,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-08T00:25:21,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742257_1433 (size=12151) 2024-11-08T00:25:21,482 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/787d05fcb686498b827264bba6fe9e26 2024-11-08T00:25:21,504 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/18476e3f6aef4ba7b23d38dbd8486667 is 50, key is test_row_0/C:col10/1731025519308/Put/seqid=0 2024-11-08T00:25:21,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742258_1434 (size=12151) 2024-11-08T00:25:21,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-08T00:25:21,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:21,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. as already flushing 2024-11-08T00:25:21,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:21,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:21,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:21,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:21,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-08T00:25:21,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:21,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. as already flushing 2024-11-08T00:25:21,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:21,766 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:21,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:21,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-08T00:25:21,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:21,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:21,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1731025581787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:21,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:21,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:21,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:21,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:21,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025581808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:21,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:21,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:21,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:21,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:21,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025581817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:21,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:21,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025581817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:21,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:21,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:21,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:21,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:21,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:21,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:21,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:21,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025581827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:21,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:21,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:21,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:21,914 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/18476e3f6aef4ba7b23d38dbd8486667 2024-11-08T00:25:21,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-08T00:25:21,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:21,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. as already flushing 2024-11-08T00:25:21,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:21,927 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:21,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:21,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:21,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/1d3a1e30ada54cf785b89fb745d5435a as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/1d3a1e30ada54cf785b89fb745d5435a 2024-11-08T00:25:21,936 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/1d3a1e30ada54cf785b89fb745d5435a, entries=200, sequenceid=248, filesize=38.8 K 2024-11-08T00:25:21,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/787d05fcb686498b827264bba6fe9e26 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/787d05fcb686498b827264bba6fe9e26 2024-11-08T00:25:21,943 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/787d05fcb686498b827264bba6fe9e26, entries=150, sequenceid=248, filesize=11.9 K 2024-11-08T00:25:21,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/18476e3f6aef4ba7b23d38dbd8486667 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/18476e3f6aef4ba7b23d38dbd8486667 2024-11-08T00:25:21,949 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/18476e3f6aef4ba7b23d38dbd8486667, entries=150, sequenceid=248, filesize=11.9 K 2024-11-08T00:25:21,949 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 0d1e33fe1a366293d5e32ce198366168 in 1447ms, sequenceid=248, compaction requested=true 2024-11-08T00:25:21,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0d1e33fe1a366293d5e32ce198366168: 2024-11-08T00:25:21,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d1e33fe1a366293d5e32ce198366168:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T00:25:21,950 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:25:21,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:25:21,950 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:25:21,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d1e33fe1a366293d5e32ce198366168:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T00:25:21,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:25:21,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d1e33fe1a366293d5e32ce198366168:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:25:21,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:25:21,951 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102403 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:25:21,951 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): 0d1e33fe1a366293d5e32ce198366168/A is initiating minor compaction (all files) 2024-11-08T00:25:21,951 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d1e33fe1a366293d5e32ce198366168/A in TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:21,951 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/5bd4292914e24444b07d971cc6d6af0f, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/791c5f4f0c4443ef8135a8bf0d633b81, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/1d3a1e30ada54cf785b89fb745d5435a] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp, totalSize=100.0 K 2024-11-08T00:25:21,951 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:21,951 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. files: [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/5bd4292914e24444b07d971cc6d6af0f, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/791c5f4f0c4443ef8135a8bf0d633b81, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/1d3a1e30ada54cf785b89fb745d5435a] 2024-11-08T00:25:21,952 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:25:21,952 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): 0d1e33fe1a366293d5e32ce198366168/B is initiating minor compaction (all files) 2024-11-08T00:25:21,952 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d1e33fe1a366293d5e32ce198366168/B in TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:21,952 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/2453a3d6405b4208b74fc357e1a04ef2, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/c1c8e298011a442a858619408a770181, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/787d05fcb686498b827264bba6fe9e26] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp, totalSize=36.0 K 2024-11-08T00:25:21,952 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5bd4292914e24444b07d971cc6d6af0f, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731025518665 2024-11-08T00:25:21,952 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 2453a3d6405b4208b74fc357e1a04ef2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731025518665 2024-11-08T00:25:21,952 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 791c5f4f0c4443ef8135a8bf0d633b81, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731025518862 2024-11-08T00:25:21,953 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d3a1e30ada54cf785b89fb745d5435a, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731025519287 2024-11-08T00:25:21,953 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting c1c8e298011a442a858619408a770181, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731025518862 2024-11-08T00:25:21,953 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 787d05fcb686498b827264bba6fe9e26, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731025519308 2024-11-08T00:25:21,961 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0d1e33fe1a366293d5e32ce198366168] 2024-11-08T00:25:21,967 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d1e33fe1a366293d5e32ce198366168#B#compaction#369 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:25:21,968 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/88176bb8ed724ae2a4d554117f1e3d65 is 50, key is test_row_0/B:col10/1731025519308/Put/seqid=0 2024-11-08T00:25:21,977 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411087cfe4b6dfb814eadb36fb476ea61ac14_0d1e33fe1a366293d5e32ce198366168 store=[table=TestAcidGuarantees family=A region=0d1e33fe1a366293d5e32ce198366168] 2024-11-08T00:25:21,979 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411087cfe4b6dfb814eadb36fb476ea61ac14_0d1e33fe1a366293d5e32ce198366168, store=[table=TestAcidGuarantees family=A region=0d1e33fe1a366293d5e32ce198366168] 2024-11-08T00:25:21,979 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411087cfe4b6dfb814eadb36fb476ea61ac14_0d1e33fe1a366293d5e32ce198366168 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0d1e33fe1a366293d5e32ce198366168] 2024-11-08T00:25:22,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742259_1435 (size=12697) 2024-11-08T00:25:22,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742260_1436 (size=4469) 2024-11-08T00:25:22,051 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/88176bb8ed724ae2a4d554117f1e3d65 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/88176bb8ed724ae2a4d554117f1e3d65 2024-11-08T00:25:22,055 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d1e33fe1a366293d5e32ce198366168/B of 0d1e33fe1a366293d5e32ce198366168 into 88176bb8ed724ae2a4d554117f1e3d65(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:25:22,055 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d1e33fe1a366293d5e32ce198366168: 2024-11-08T00:25:22,055 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., storeName=0d1e33fe1a366293d5e32ce198366168/B, priority=13, startTime=1731025521950; duration=0sec 2024-11-08T00:25:22,055 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:25:22,055 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d1e33fe1a366293d5e32ce198366168:B 2024-11-08T00:25:22,055 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:25:22,056 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:25:22,056 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): 0d1e33fe1a366293d5e32ce198366168/C is initiating minor compaction (all files) 2024-11-08T00:25:22,056 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d1e33fe1a366293d5e32ce198366168/C in TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:22,056 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/a570989d8d124566bec065cd6fe4f9d4, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/06b10b58ddc4474c950e6fb116dbbc86, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/18476e3f6aef4ba7b23d38dbd8486667] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp, totalSize=36.0 K 2024-11-08T00:25:22,057 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting a570989d8d124566bec065cd6fe4f9d4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731025518665 2024-11-08T00:25:22,057 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 06b10b58ddc4474c950e6fb116dbbc86, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731025518862 2024-11-08T00:25:22,057 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 18476e3f6aef4ba7b23d38dbd8486667, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731025519308 2024-11-08T00:25:22,074 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d1e33fe1a366293d5e32ce198366168#C#compaction#370 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:25:22,075 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/a0257f17ee3241a787dac005f8134dc2 is 50, key is test_row_0/C:col10/1731025519308/Put/seqid=0 2024-11-08T00:25:22,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-08T00:25:22,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:22,085 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 0d1e33fe1a366293d5e32ce198366168 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-08T00:25:22,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742261_1437 (size=12697) 2024-11-08T00:25:22,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108b257c660cf19402c84b89e70731046dd_0d1e33fe1a366293d5e32ce198366168 is 50, key is test_row_0/A:col10/1731025520596/Put/seqid=0 2024-11-08T00:25:22,122 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/a0257f17ee3241a787dac005f8134dc2 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/a0257f17ee3241a787dac005f8134dc2 2024-11-08T00:25:22,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742262_1438 (size=12454) 2024-11-08T00:25:22,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,131 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108b257c660cf19402c84b89e70731046dd_0d1e33fe1a366293d5e32ce198366168 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108b257c660cf19402c84b89e70731046dd_0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:22,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/0a5d6b933542435e9bdfd431a48e3b9f, store: [table=TestAcidGuarantees family=A region=0d1e33fe1a366293d5e32ce198366168] 2024-11-08T00:25:22,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/0a5d6b933542435e9bdfd431a48e3b9f is 175, key is test_row_0/A:col10/1731025520596/Put/seqid=0 2024-11-08T00:25:22,132 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d1e33fe1a366293d5e32ce198366168/C of 0d1e33fe1a366293d5e32ce198366168 into a0257f17ee3241a787dac005f8134dc2(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:25:22,132 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d1e33fe1a366293d5e32ce198366168: 2024-11-08T00:25:22,132 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., storeName=0d1e33fe1a366293d5e32ce198366168/C, priority=13, startTime=1731025521950; duration=0sec 2024-11-08T00:25:22,132 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:25:22,133 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d1e33fe1a366293d5e32ce198366168:C 2024-11-08T00:25:22,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742263_1439 (size=31255) 2024-11-08T00:25:22,165 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=268, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/0a5d6b933542435e9bdfd431a48e3b9f 2024-11-08T00:25:22,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/58913a3b9c25496ca6cf638d1e08c914 is 50, key is test_row_0/B:col10/1731025520596/Put/seqid=0 2024-11-08T00:25:22,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742264_1440 (size=12301) 2024-11-08T00:25:22,239 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/58913a3b9c25496ca6cf638d1e08c914 2024-11-08T00:25:22,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/45d6270537ad466499d8e86c6691722e is 50, key is test_row_0/C:col10/1731025520596/Put/seqid=0 2024-11-08T00:25:22,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-08T00:25:22,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742265_1441 (size=12301) 2024-11-08T00:25:22,299 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/45d6270537ad466499d8e86c6691722e 2024-11-08T00:25:22,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/0a5d6b933542435e9bdfd431a48e3b9f as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/0a5d6b933542435e9bdfd431a48e3b9f 2024-11-08T00:25:22,310 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/0a5d6b933542435e9bdfd431a48e3b9f, entries=150, sequenceid=268, filesize=30.5 K 2024-11-08T00:25:22,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/58913a3b9c25496ca6cf638d1e08c914 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/58913a3b9c25496ca6cf638d1e08c914 2024-11-08T00:25:22,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,315 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/58913a3b9c25496ca6cf638d1e08c914, entries=150, sequenceid=268, filesize=12.0 K 2024-11-08T00:25:22,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/45d6270537ad466499d8e86c6691722e as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/45d6270537ad466499d8e86c6691722e 2024-11-08T00:25:22,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,323 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/45d6270537ad466499d8e86c6691722e, entries=150, sequenceid=268, filesize=12.0 K 2024-11-08T00:25:22,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,324 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=0 B/0 for 0d1e33fe1a366293d5e32ce198366168 in 239ms, sequenceid=268, compaction requested=false 2024-11-08T00:25:22,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 0d1e33fe1a366293d5e32ce198366168: 2024-11-08T00:25:22,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:22,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-08T00:25:22,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-08T00:25:22,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,329 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-08T00:25:22,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,329 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1810 sec 2024-11-08T00:25:22,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,331 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.1890 sec 2024-11-08T00:25:22,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,453 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d1e33fe1a366293d5e32ce198366168#A#compaction#368 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:25:22,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,454 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/9818dd23e06744c2b3553303d72882a4 is 175, key is test_row_0/A:col10/1731025519308/Put/seqid=0 2024-11-08T00:25:22,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742266_1442 (size=31651) 2024-11-08T00:25:22,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,914 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/9818dd23e06744c2b3553303d72882a4 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/9818dd23e06744c2b3553303d72882a4 2024-11-08T00:25:22,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,922 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d1e33fe1a366293d5e32ce198366168/A of 0d1e33fe1a366293d5e32ce198366168 into 9818dd23e06744c2b3553303d72882a4(size=30.9 K), total size for store is 61.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:25:22,922 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d1e33fe1a366293d5e32ce198366168: 2024-11-08T00:25:22,922 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., storeName=0d1e33fe1a366293d5e32ce198366168/A, priority=13, startTime=1731025521950; duration=0sec 2024-11-08T00:25:22,922 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:25:22,922 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d1e33fe1a366293d5e32ce198366168:A 2024-11-08T00:25:22,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,986 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:25:22,986 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0d1e33fe1a366293d5e32ce198366168 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T00:25:22,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:22,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:22,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:23,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:23,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:23,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:23,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:23,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:23,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:23,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:23,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:23,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:23,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:23,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:23,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:23,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:23,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:23,019 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110801fec44bc1e34f56b5e367de30a7ade2_0d1e33fe1a366293d5e32ce198366168 is 50, key is test_row_0/A:col10/1731025522980/Put/seqid=0 2024-11-08T00:25:23,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:23,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:23,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:23,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:23,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:23,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:23,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:23,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:23,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742267_1443 (size=20074) 2024-11-08T00:25:23,074 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:23,081 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110801fec44bc1e34f56b5e367de30a7ade2_0d1e33fe1a366293d5e32ce198366168 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110801fec44bc1e34f56b5e367de30a7ade2_0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:23,082 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/9db1b696a60b4ab0a27eb28180b81c09, store: [table=TestAcidGuarantees family=A region=0d1e33fe1a366293d5e32ce198366168] 2024-11-08T00:25:23,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/9db1b696a60b4ab0a27eb28180b81c09 is 175, key is test_row_0/A:col10/1731025522980/Put/seqid=0 2024-11-08T00:25:23,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:23,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025583082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:23,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:23,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:23,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:23,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025583086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:23,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:23,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:23,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:23,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:23,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:23,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025583100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:23,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:23,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:23,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:23,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742268_1444 (size=57333) 2024-11-08T00:25:23,118 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=281, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/9db1b696a60b4ab0a27eb28180b81c09 2024-11-08T00:25:23,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:23,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1731025583104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:23,122 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:23,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:23,122 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:23,122 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:23,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025583104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:23,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:23,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:23,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:23,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/2935adb519044e94b13b50703dc54387 is 50, key is test_row_0/B:col10/1731025522980/Put/seqid=0 2024-11-08T00:25:23,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742269_1445 (size=12301) 2024-11-08T00:25:23,139 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/2935adb519044e94b13b50703dc54387 2024-11-08T00:25:23,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/6b571238b4dd40419bf19a14b3a0c863 is 50, key is test_row_0/C:col10/1731025522980/Put/seqid=0 2024-11-08T00:25:23,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742270_1446 (size=12301) 2024-11-08T00:25:23,158 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/6b571238b4dd40419bf19a14b3a0c863 2024-11-08T00:25:23,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/9db1b696a60b4ab0a27eb28180b81c09 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/9db1b696a60b4ab0a27eb28180b81c09 2024-11-08T00:25:23,170 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/9db1b696a60b4ab0a27eb28180b81c09, entries=300, sequenceid=281, filesize=56.0 K 2024-11-08T00:25:23,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/2935adb519044e94b13b50703dc54387 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/2935adb519044e94b13b50703dc54387 2024-11-08T00:25:23,201 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/2935adb519044e94b13b50703dc54387, entries=150, sequenceid=281, filesize=12.0 K 2024-11-08T00:25:23,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/6b571238b4dd40419bf19a14b3a0c863 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/6b571238b4dd40419bf19a14b3a0c863 2024-11-08T00:25:23,206 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/6b571238b4dd40419bf19a14b3a0c863, entries=150, sequenceid=281, filesize=12.0 K 2024-11-08T00:25:23,207 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 0d1e33fe1a366293d5e32ce198366168 in 221ms, sequenceid=281, compaction requested=true 2024-11-08T00:25:23,207 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0d1e33fe1a366293d5e32ce198366168: 2024-11-08T00:25:23,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d1e33fe1a366293d5e32ce198366168:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T00:25:23,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:25:23,207 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:25:23,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d1e33fe1a366293d5e32ce198366168:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T00:25:23,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:25:23,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d1e33fe1a366293d5e32ce198366168:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:25:23,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-08T00:25:23,207 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:25:23,208 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 120239 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:25:23,208 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:25:23,208 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): 0d1e33fe1a366293d5e32ce198366168/B is initiating minor compaction (all files) 2024-11-08T00:25:23,208 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): 0d1e33fe1a366293d5e32ce198366168/A is initiating minor compaction (all files) 2024-11-08T00:25:23,208 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d1e33fe1a366293d5e32ce198366168/A in TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:23,208 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d1e33fe1a366293d5e32ce198366168/B in TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:23,208 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/9818dd23e06744c2b3553303d72882a4, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/0a5d6b933542435e9bdfd431a48e3b9f, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/9db1b696a60b4ab0a27eb28180b81c09] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp, totalSize=117.4 K 2024-11-08T00:25:23,208 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/88176bb8ed724ae2a4d554117f1e3d65, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/58913a3b9c25496ca6cf638d1e08c914, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/2935adb519044e94b13b50703dc54387] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp, totalSize=36.4 K 2024-11-08T00:25:23,208 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:23,209 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. files: [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/9818dd23e06744c2b3553303d72882a4, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/0a5d6b933542435e9bdfd431a48e3b9f, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/9db1b696a60b4ab0a27eb28180b81c09] 2024-11-08T00:25:23,209 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 88176bb8ed724ae2a4d554117f1e3d65, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731025519308 2024-11-08T00:25:23,209 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9818dd23e06744c2b3553303d72882a4, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731025519308 2024-11-08T00:25:23,209 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 58913a3b9c25496ca6cf638d1e08c914, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1731025520596 2024-11-08T00:25:23,211 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a5d6b933542435e9bdfd431a48e3b9f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1731025520596 2024-11-08T00:25:23,212 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 2935adb519044e94b13b50703dc54387, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731025522954 2024-11-08T00:25:23,214 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9db1b696a60b4ab0a27eb28180b81c09, keycount=300, bloomtype=ROW, size=56.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731025522954 2024-11-08T00:25:23,217 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:25:23,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:23,217 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0d1e33fe1a366293d5e32ce198366168 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-08T00:25:23,232 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d1e33fe1a366293d5e32ce198366168#B#compaction#377 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:25:23,233 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/e3210d33664c4bb2838ea6bb1c9888f4 is 50, key is test_row_0/B:col10/1731025522980/Put/seqid=0 2024-11-08T00:25:23,241 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0d1e33fe1a366293d5e32ce198366168] 2024-11-08T00:25:23,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:23,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025583233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:23,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:23,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025583234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:23,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:23,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:23,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1731025583239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:23,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:23,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:23,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:23,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025583237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:23,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:23,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025583245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:23,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:23,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:23,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:23,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:23,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:23,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:23,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:23,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:23,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:23,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:23,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:23,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:23,277 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110806a62b7cf28346ad8b4578b267ed2d52_0d1e33fe1a366293d5e32ce198366168 is 50, key is test_row_0/A:col10/1731025523081/Put/seqid=0 2024-11-08T00:25:23,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-08T00:25:23,287 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:25:23,288 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-08T00:25:23,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-08T00:25:23,293 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241108c121cedda0154dd4a79dc45d62a0db56_0d1e33fe1a366293d5e32ce198366168 store=[table=TestAcidGuarantees family=A region=0d1e33fe1a366293d5e32ce198366168] 2024-11-08T00:25:23,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-08T00:25:23,294 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T00:25:23,295 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T00:25:23,295 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T00:25:23,296 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241108c121cedda0154dd4a79dc45d62a0db56_0d1e33fe1a366293d5e32ce198366168, store=[table=TestAcidGuarantees family=A region=0d1e33fe1a366293d5e32ce198366168] 2024-11-08T00:25:23,296 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108c121cedda0154dd4a79dc45d62a0db56_0d1e33fe1a366293d5e32ce198366168 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0d1e33fe1a366293d5e32ce198366168] 2024-11-08T00:25:23,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742271_1447 (size=12949) 2024-11-08T00:25:23,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742273_1449 (size=4469) 2024-11-08T00:25:23,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742272_1448 (size=14994) 2024-11-08T00:25:23,347 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:23,350 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110806a62b7cf28346ad8b4578b267ed2d52_0d1e33fe1a366293d5e32ce198366168 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110806a62b7cf28346ad8b4578b267ed2d52_0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:23,351 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/ba6a47187d6542188130ed865992a3ee, store: [table=TestAcidGuarantees family=A region=0d1e33fe1a366293d5e32ce198366168] 2024-11-08T00:25:23,351 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/ba6a47187d6542188130ed865992a3ee is 175, key is test_row_0/A:col10/1731025523081/Put/seqid=0 2024-11-08T00:25:23,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742274_1450 (size=39949) 2024-11-08T00:25:23,364 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=308, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/ba6a47187d6542188130ed865992a3ee 2024-11-08T00:25:23,375 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/7e51bd399c694121b785c3237222d6d0 is 50, key is test_row_0/B:col10/1731025523081/Put/seqid=0 2024-11-08T00:25:23,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:23,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025583367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:23,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:23,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025583367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:23,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:23,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:23,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:23,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1731025583367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:23,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:23,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:23,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:23,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:23,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025583367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:23,377 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:23,377 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:23,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:23,377 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:23,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:23,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:23,377 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:23,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:23,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025583367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:23,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:23,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:23,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:23,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742275_1451 (size=12301) 2024-11-08T00:25:23,395 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/7e51bd399c694121b785c3237222d6d0 2024-11-08T00:25:23,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-08T00:25:23,407 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/86cf1460b3654785b01bc5057207e2c0 is 50, key is test_row_0/C:col10/1731025523081/Put/seqid=0 2024-11-08T00:25:23,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742276_1452 (size=12301) 2024-11-08T00:25:23,445 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/86cf1460b3654785b01bc5057207e2c0 2024-11-08T00:25:23,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-08T00:25:23,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:23,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. as already flushing 2024-11-08T00:25:23,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:23,448 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:23,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:23,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:23,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/ba6a47187d6542188130ed865992a3ee as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/ba6a47187d6542188130ed865992a3ee 2024-11-08T00:25:23,466 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/ba6a47187d6542188130ed865992a3ee, entries=200, sequenceid=308, filesize=39.0 K 2024-11-08T00:25:23,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/7e51bd399c694121b785c3237222d6d0 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/7e51bd399c694121b785c3237222d6d0 2024-11-08T00:25:23,480 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/7e51bd399c694121b785c3237222d6d0, entries=150, sequenceid=308, filesize=12.0 K 2024-11-08T00:25:23,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/86cf1460b3654785b01bc5057207e2c0 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/86cf1460b3654785b01bc5057207e2c0 2024-11-08T00:25:23,491 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/86cf1460b3654785b01bc5057207e2c0, entries=150, sequenceid=308, filesize=12.0 K 2024-11-08T00:25:23,492 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 0d1e33fe1a366293d5e32ce198366168 in 274ms, sequenceid=308, compaction requested=true 2024-11-08T00:25:23,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0d1e33fe1a366293d5e32ce198366168: 2024-11-08T00:25:23,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d1e33fe1a366293d5e32ce198366168:A, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:25:23,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-08T00:25:23,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d1e33fe1a366293d5e32ce198366168:B, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:25:23,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-08T00:25:23,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d1e33fe1a366293d5e32ce198366168:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:25:23,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-08T00:25:23,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:23,596 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:25:23,596 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0d1e33fe1a366293d5e32ce198366168 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T00:25:23,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-08T00:25:23,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:23,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. as already flushing 2024-11-08T00:25:23,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:23,602 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:23,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:23,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:23,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-08T00:25:23,615 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110878fa4cb07b1d476bb0098ca3fc60b0a3_0d1e33fe1a366293d5e32ce198366168 is 50, key is test_row_0/A:col10/1731025523595/Put/seqid=0 2024-11-08T00:25:23,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742277_1453 (size=20074) 2024-11-08T00:25:23,634 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:23,641 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110878fa4cb07b1d476bb0098ca3fc60b0a3_0d1e33fe1a366293d5e32ce198366168 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110878fa4cb07b1d476bb0098ca3fc60b0a3_0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:23,643 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/b5a7bb56b6f046e0bafb3bb589033458, store: [table=TestAcidGuarantees family=A region=0d1e33fe1a366293d5e32ce198366168] 2024-11-08T00:25:23,643 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/b5a7bb56b6f046e0bafb3bb589033458 is 175, key is test_row_0/A:col10/1731025523595/Put/seqid=0 2024-11-08T00:25:23,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742278_1454 (size=57333) 2024-11-08T00:25:23,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:23,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025583663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:23,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:23,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:23,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:23,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:23,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025583665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:23,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:23,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:23,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025583679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:23,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:23,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:23,686 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:23,686 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:23,686 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:23,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:23,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025583680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:23,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:23,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1731025583680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:23,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:23,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:23,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:23,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:23,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:23,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:23,739 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/e3210d33664c4bb2838ea6bb1c9888f4 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/e3210d33664c4bb2838ea6bb1c9888f4 2024-11-08T00:25:23,739 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d1e33fe1a366293d5e32ce198366168#A#compaction#378 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:25:23,739 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/a9c9c7505b0f430590394fca8386bd31 is 175, key is test_row_0/A:col10/1731025522980/Put/seqid=0 2024-11-08T00:25:23,745 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d1e33fe1a366293d5e32ce198366168/B of 0d1e33fe1a366293d5e32ce198366168 into e3210d33664c4bb2838ea6bb1c9888f4(size=12.6 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:25:23,745 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d1e33fe1a366293d5e32ce198366168: 2024-11-08T00:25:23,745 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., storeName=0d1e33fe1a366293d5e32ce198366168/B, priority=13, startTime=1731025523207; duration=0sec 2024-11-08T00:25:23,745 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-08T00:25:23,746 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d1e33fe1a366293d5e32ce198366168:B 2024-11-08T00:25:23,746 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 3 compacting, 1 eligible, 16 blocking 2024-11-08T00:25:23,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742279_1455 (size=31903) 2024-11-08T00:25:23,746 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-08T00:25:23,746 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-08T00:25:23,746 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. because compaction request was cancelled 2024-11-08T00:25:23,746 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d1e33fe1a366293d5e32ce198366168:A 2024-11-08T00:25:23,746 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T00:25:23,747 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49600 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T00:25:23,747 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): 0d1e33fe1a366293d5e32ce198366168/C is initiating minor compaction (all files) 2024-11-08T00:25:23,747 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d1e33fe1a366293d5e32ce198366168/C in TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:23,748 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/a0257f17ee3241a787dac005f8134dc2, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/45d6270537ad466499d8e86c6691722e, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/6b571238b4dd40419bf19a14b3a0c863, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/86cf1460b3654785b01bc5057207e2c0] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp, totalSize=48.4 K 2024-11-08T00:25:23,748 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting a0257f17ee3241a787dac005f8134dc2, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731025519308 2024-11-08T00:25:23,748 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 45d6270537ad466499d8e86c6691722e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1731025520596 2024-11-08T00:25:23,748 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b571238b4dd40419bf19a14b3a0c863, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731025522954 2024-11-08T00:25:23,749 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 86cf1460b3654785b01bc5057207e2c0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731025523081 2024-11-08T00:25:23,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-08T00:25:23,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:23,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. as already flushing 2024-11-08T00:25:23,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:23,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:23,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:23,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:23,782 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d1e33fe1a366293d5e32ce198366168#C#compaction#383 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:25:23,783 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/01d8d4151e754493a4cd5dfce3c4a90f is 50, key is test_row_0/C:col10/1731025523081/Put/seqid=0 2024-11-08T00:25:23,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742280_1456 (size=12983) 2024-11-08T00:25:23,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:23,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025583791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:23,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:23,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:23,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:23,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:23,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025583797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:23,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:23,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025583797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:23,810 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:23,810 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:23,810 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:23,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:23,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:23,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:23,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:23,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1731025583808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:23,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:23,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025583808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:23,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:23,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:23,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:23,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:23,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:23,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:23,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-08T00:25:23,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:23,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. as already flushing 2024-11-08T00:25:23,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:23,917 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:23,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:23,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-08T00:25:23,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:24,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:24,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025584017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:24,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:24,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:24,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:24,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025584017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:24,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:24,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:24,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025584017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:24,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:24,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:24,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:24,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:24,023 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:24,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025584017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:24,023 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:24,023 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:24,023 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:24,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:24,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:24,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:24,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1731025584027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:24,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:24,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:24,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:24,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-08T00:25:24,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:24,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. as already flushing 2024-11-08T00:25:24,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:24,072 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:24,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:24,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:24,075 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=320, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/b5a7bb56b6f046e0bafb3bb589033458 2024-11-08T00:25:24,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/771fbd0a19264e53a16949e62aa5552d is 50, key is test_row_0/B:col10/1731025523595/Put/seqid=0 2024-11-08T00:25:24,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742281_1457 (size=12301) 2024-11-08T00:25:24,093 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/771fbd0a19264e53a16949e62aa5552d 2024-11-08T00:25:24,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/91484d2347c64dad8fa0a8db38142a23 is 50, key is test_row_0/C:col10/1731025523595/Put/seqid=0 2024-11-08T00:25:24,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742282_1458 (size=12301) 2024-11-08T00:25:24,151 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/a9c9c7505b0f430590394fca8386bd31 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/a9c9c7505b0f430590394fca8386bd31 2024-11-08T00:25:24,157 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d1e33fe1a366293d5e32ce198366168/A of 0d1e33fe1a366293d5e32ce198366168 into a9c9c7505b0f430590394fca8386bd31(size=31.2 K), total size for store is 70.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:25:24,157 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d1e33fe1a366293d5e32ce198366168: 2024-11-08T00:25:24,157 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., storeName=0d1e33fe1a366293d5e32ce198366168/A, priority=13, startTime=1731025523207; duration=0sec 2024-11-08T00:25:24,157 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-08T00:25:24,157 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d1e33fe1a366293d5e32ce198366168:A 2024-11-08T00:25:24,157 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d1e33fe1a366293d5e32ce198366168:B 2024-11-08T00:25:24,157 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 4 compacting, 0 eligible, 16 blocking 2024-11-08T00:25:24,157 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-08T00:25:24,157 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-08T00:25:24,158 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. because compaction request was cancelled 2024-11-08T00:25:24,158 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d1e33fe1a366293d5e32ce198366168:C 2024-11-08T00:25:24,158 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-08T00:25:24,158 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-08T00:25:24,158 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-08T00:25:24,158 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. because compaction request was cancelled 2024-11-08T00:25:24,158 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d1e33fe1a366293d5e32ce198366168:B 2024-11-08T00:25:24,210 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/01d8d4151e754493a4cd5dfce3c4a90f as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/01d8d4151e754493a4cd5dfce3c4a90f 2024-11-08T00:25:24,214 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0d1e33fe1a366293d5e32ce198366168/C of 0d1e33fe1a366293d5e32ce198366168 into 01d8d4151e754493a4cd5dfce3c4a90f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:25:24,214 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d1e33fe1a366293d5e32ce198366168: 2024-11-08T00:25:24,214 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., storeName=0d1e33fe1a366293d5e32ce198366168/C, priority=12, startTime=1731025523492; duration=0sec 2024-11-08T00:25:24,215 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:25:24,215 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d1e33fe1a366293d5e32ce198366168:C 2024-11-08T00:25:24,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-08T00:25:24,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:24,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. as already flushing 2024-11-08T00:25:24,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:24,225 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:24,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:24,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:24,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:24,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025584328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:24,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:24,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025584328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:24,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:24,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:24,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:24,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:24,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025584328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:24,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:24,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:24,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:24,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:24,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:24,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:24,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025584328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:24,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:24,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:24,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:24,335 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:24,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:24,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1731025584347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:24,357 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:24,357 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:24,357 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:24,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-08T00:25:24,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:24,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. as already flushing 2024-11-08T00:25:24,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:24,381 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:24,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:24,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:24,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-08T00:25:24,507 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/91484d2347c64dad8fa0a8db38142a23 2024-11-08T00:25:24,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/b5a7bb56b6f046e0bafb3bb589033458 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/b5a7bb56b6f046e0bafb3bb589033458 2024-11-08T00:25:24,518 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/b5a7bb56b6f046e0bafb3bb589033458, entries=300, sequenceid=320, filesize=56.0 K 2024-11-08T00:25:24,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/771fbd0a19264e53a16949e62aa5552d as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/771fbd0a19264e53a16949e62aa5552d 2024-11-08T00:25:24,530 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/771fbd0a19264e53a16949e62aa5552d, entries=150, sequenceid=320, filesize=12.0 K 2024-11-08T00:25:24,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/91484d2347c64dad8fa0a8db38142a23 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/91484d2347c64dad8fa0a8db38142a23 2024-11-08T00:25:24,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-08T00:25:24,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:24,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. as already flushing 2024-11-08T00:25:24,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:24,535 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:24,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:24,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:24,538 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/91484d2347c64dad8fa0a8db38142a23, entries=150, sequenceid=320, filesize=12.0 K 2024-11-08T00:25:24,543 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 0d1e33fe1a366293d5e32ce198366168 in 947ms, sequenceid=320, compaction requested=true 2024-11-08T00:25:24,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0d1e33fe1a366293d5e32ce198366168: 2024-11-08T00:25:24,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d1e33fe1a366293d5e32ce198366168:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T00:25:24,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:25:24,544 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:25:24,544 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:25:24,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d1e33fe1a366293d5e32ce198366168:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T00:25:24,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:25:24,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d1e33fe1a366293d5e32ce198366168:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:25:24,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:25:24,545 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:25:24,545 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 129185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:25:24,545 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): 0d1e33fe1a366293d5e32ce198366168/A is initiating minor compaction (all files) 2024-11-08T00:25:24,545 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): 0d1e33fe1a366293d5e32ce198366168/B is initiating minor compaction (all files) 2024-11-08T00:25:24,545 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d1e33fe1a366293d5e32ce198366168/B in TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:24,545 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d1e33fe1a366293d5e32ce198366168/A in TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:24,545 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/a9c9c7505b0f430590394fca8386bd31, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/ba6a47187d6542188130ed865992a3ee, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/b5a7bb56b6f046e0bafb3bb589033458] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp, totalSize=126.2 K 2024-11-08T00:25:24,545 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/e3210d33664c4bb2838ea6bb1c9888f4, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/7e51bd399c694121b785c3237222d6d0, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/771fbd0a19264e53a16949e62aa5552d] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp, totalSize=36.7 K 2024-11-08T00:25:24,545 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:24,545 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. files: [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/a9c9c7505b0f430590394fca8386bd31, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/ba6a47187d6542188130ed865992a3ee, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/b5a7bb56b6f046e0bafb3bb589033458] 2024-11-08T00:25:24,545 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting e3210d33664c4bb2838ea6bb1c9888f4, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731025522954 2024-11-08T00:25:24,545 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9c9c7505b0f430590394fca8386bd31, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731025522954 2024-11-08T00:25:24,546 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba6a47187d6542188130ed865992a3ee, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731025523081 2024-11-08T00:25:24,546 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e51bd399c694121b785c3237222d6d0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731025523081 2024-11-08T00:25:24,546 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 771fbd0a19264e53a16949e62aa5552d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731025523592 2024-11-08T00:25:24,546 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5a7bb56b6f046e0bafb3bb589033458, keycount=300, bloomtype=ROW, size=56.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731025523233 2024-11-08T00:25:24,566 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d1e33fe1a366293d5e32ce198366168#B#compaction#386 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:25:24,567 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/05ad2918b67b4840937dd29168b0f6f6 is 50, key is test_row_0/B:col10/1731025523595/Put/seqid=0 2024-11-08T00:25:24,572 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0d1e33fe1a366293d5e32ce198366168] 2024-11-08T00:25:24,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742283_1459 (size=13051) 2024-11-08T00:25:24,585 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241108982d4ef62a0b439db7b6c4cf734a5432_0d1e33fe1a366293d5e32ce198366168 store=[table=TestAcidGuarantees family=A region=0d1e33fe1a366293d5e32ce198366168] 2024-11-08T00:25:24,587 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241108982d4ef62a0b439db7b6c4cf734a5432_0d1e33fe1a366293d5e32ce198366168, store=[table=TestAcidGuarantees family=A region=0d1e33fe1a366293d5e32ce198366168] 2024-11-08T00:25:24,587 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108982d4ef62a0b439db7b6c4cf734a5432_0d1e33fe1a366293d5e32ce198366168 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0d1e33fe1a366293d5e32ce198366168] 2024-11-08T00:25:24,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742284_1460 (size=4469) 2024-11-08T00:25:24,618 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d1e33fe1a366293d5e32ce198366168#A#compaction#387 average throughput is 0.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:25:24,619 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/321752b8ea214a9f96fdf1f6ff4f95a4 is 175, key is test_row_0/A:col10/1731025523595/Put/seqid=0 2024-11-08T00:25:24,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742285_1461 (size=32005) 2024-11-08T00:25:24,653 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/321752b8ea214a9f96fdf1f6ff4f95a4 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/321752b8ea214a9f96fdf1f6ff4f95a4 2024-11-08T00:25:24,658 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d1e33fe1a366293d5e32ce198366168/A of 0d1e33fe1a366293d5e32ce198366168 into 321752b8ea214a9f96fdf1f6ff4f95a4(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:25:24,658 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d1e33fe1a366293d5e32ce198366168: 2024-11-08T00:25:24,658 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., storeName=0d1e33fe1a366293d5e32ce198366168/A, priority=13, startTime=1731025524543; duration=0sec 2024-11-08T00:25:24,658 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:25:24,658 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d1e33fe1a366293d5e32ce198366168:A 2024-11-08T00:25:24,659 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-08T00:25:24,659 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-08T00:25:24,659 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-08T00:25:24,659 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. because compaction request was cancelled 2024-11-08T00:25:24,659 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d1e33fe1a366293d5e32ce198366168:C 2024-11-08T00:25:24,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-08T00:25:24,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:24,689 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 0d1e33fe1a366293d5e32ce198366168 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-08T00:25:24,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108c3b501476f5a43e7adb66446fb1fb5db_0d1e33fe1a366293d5e32ce198366168 is 50, key is test_row_0/A:col10/1731025523677/Put/seqid=0 2024-11-08T00:25:24,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742286_1462 (size=12454) 2024-11-08T00:25:24,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. as already flushing 2024-11-08T00:25:24,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:24,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:24,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025584867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:24,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:24,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:24,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1731025584867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:24,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:24,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:24,873 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:24,873 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:24,873 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:24,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:24,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025584869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:24,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:24,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025584871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:24,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:24,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:24,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:24,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:24,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025584873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:24,883 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:24,883 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:24,883 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:24,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:24,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:24,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:24,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:24,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025584977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:24,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:24,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:24,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:24,988 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/05ad2918b67b4840937dd29168b0f6f6 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/05ad2918b67b4840937dd29168b0f6f6 2024-11-08T00:25:24,992 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d1e33fe1a366293d5e32ce198366168/B of 0d1e33fe1a366293d5e32ce198366168 into 05ad2918b67b4840937dd29168b0f6f6(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:25:24,992 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d1e33fe1a366293d5e32ce198366168: 2024-11-08T00:25:24,992 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., storeName=0d1e33fe1a366293d5e32ce198366168/B, priority=13, startTime=1731025524544; duration=0sec 2024-11-08T00:25:24,992 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:25:24,992 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d1e33fe1a366293d5e32ce198366168:B 2024-11-08T00:25:24,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:24,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025584987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:24,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:24,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:24,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025584988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:24,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:24,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025584988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:24,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:24,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:24,996 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:24,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:24,996 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:24,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:24,996 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:24,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:25,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:25,114 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108c3b501476f5a43e7adb66446fb1fb5db_0d1e33fe1a366293d5e32ce198366168 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108c3b501476f5a43e7adb66446fb1fb5db_0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:25,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/cb7b7d4d1a124c8eae05b690aab3d1b2, store: [table=TestAcidGuarantees family=A region=0d1e33fe1a366293d5e32ce198366168] 2024-11-08T00:25:25,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/cb7b7d4d1a124c8eae05b690aab3d1b2 is 175, key is test_row_0/A:col10/1731025523677/Put/seqid=0 2024-11-08T00:25:25,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742287_1463 (size=31255) 2024-11-08T00:25:25,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:25,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025585188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:25,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:25,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:25,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:25,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:25,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025585207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:25,208 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:25,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:25,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:25,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:25,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025585207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:25,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:25,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:25,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:25,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:25,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025585207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:25,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:25,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:25,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:25,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-08T00:25:25,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:25,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025585507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:25,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:25,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:25,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:25,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:25,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025585517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:25,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:25,525 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:25,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025585517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:25,525 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:25,525 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:25,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:25,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:25,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:25,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:25,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025585517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:25,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:25,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:25,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:25,529 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=348, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/cb7b7d4d1a124c8eae05b690aab3d1b2 2024-11-08T00:25:25,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/7ed8f60b900144088a51135fbb7ff9e2 is 50, key is test_row_0/B:col10/1731025523677/Put/seqid=0 2024-11-08T00:25:25,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742288_1464 (size=12301) 2024-11-08T00:25:25,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:25,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1731025585877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:25,881 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 3302f0f507bd,40479,1731025415856 for 'test_row_1' in 0d1e33fe1a366293d5e32ce198366168 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T00:25:25,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:25,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:25,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:25,946 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/7ed8f60b900144088a51135fbb7ff9e2 2024-11-08T00:25:25,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/939518c2cffd4893a17d4ed07eb33787 is 50, key is test_row_0/C:col10/1731025523677/Put/seqid=0 2024-11-08T00:25:25,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742289_1465 (size=12301) 2024-11-08T00:25:25,964 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/939518c2cffd4893a17d4ed07eb33787 2024-11-08T00:25:25,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/cb7b7d4d1a124c8eae05b690aab3d1b2 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/cb7b7d4d1a124c8eae05b690aab3d1b2 2024-11-08T00:25:25,973 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/cb7b7d4d1a124c8eae05b690aab3d1b2, entries=150, sequenceid=348, filesize=30.5 K 2024-11-08T00:25:25,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/7ed8f60b900144088a51135fbb7ff9e2 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/7ed8f60b900144088a51135fbb7ff9e2 2024-11-08T00:25:25,981 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/7ed8f60b900144088a51135fbb7ff9e2, entries=150, sequenceid=348, filesize=12.0 K 2024-11-08T00:25:25,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.StoreScanner(997): StoreScanner already has the close lock. There is no need to updateReaders 2024-11-08T00:25:25,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/939518c2cffd4893a17d4ed07eb33787 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/939518c2cffd4893a17d4ed07eb33787 2024-11-08T00:25:25,986 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/939518c2cffd4893a17d4ed07eb33787, entries=150, sequenceid=348, filesize=12.0 K 2024-11-08T00:25:25,987 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 0d1e33fe1a366293d5e32ce198366168 in 1297ms, sequenceid=348, compaction requested=true 2024-11-08T00:25:25,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 0d1e33fe1a366293d5e32ce198366168: 2024-11-08T00:25:25,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:25,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-08T00:25:25,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-08T00:25:25,989 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-08T00:25:25,990 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6930 sec 2024-11-08T00:25:25,993 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 2.7020 sec 2024-11-08T00:25:26,032 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:25:26,032 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0d1e33fe1a366293d5e32ce198366168 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-08T00:25:26,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:26,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108122fb3faefdc47c0b68ba5a072befa78_0d1e33fe1a366293d5e32ce198366168 is 50, key is test_row_0/A:col10/1731025526031/Put/seqid=0 2024-11-08T00:25:26,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742290_1466 (size=17534) 2024-11-08T00:25:26,057 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:26,061 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108122fb3faefdc47c0b68ba5a072befa78_0d1e33fe1a366293d5e32ce198366168 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108122fb3faefdc47c0b68ba5a072befa78_0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:26,062 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/c5a9ebf28fe348389cd178d167fddb4b, store: [table=TestAcidGuarantees family=A region=0d1e33fe1a366293d5e32ce198366168] 2024-11-08T00:25:26,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/c5a9ebf28fe348389cd178d167fddb4b is 175, key is test_row_0/A:col10/1731025526031/Put/seqid=0 2024-11-08T00:25:26,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742291_1467 (size=48639) 2024-11-08T00:25:26,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:26,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025586087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:26,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:26,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025586088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:26,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:26,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:26,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:26,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:26,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:26,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:26,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:26,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025586089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:26,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:26,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:26,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:26,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:26,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025586091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:26,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:26,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:26,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:26,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:26,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025586207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:26,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:26,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:26,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:26,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:26,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025586207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:26,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:26,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:26,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:26,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:26,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025586207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:26,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:26,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:26,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:26,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:26,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025586208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:26,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:26,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:26,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:26,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:26,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025586417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:26,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:26,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:26,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:26,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025586417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:26,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:26,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:26,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:26,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:26,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:26,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025586417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:26,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:26,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:26,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:26,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:26,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025586417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:26,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:26,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:26,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:26,474 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=364, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/c5a9ebf28fe348389cd178d167fddb4b 2024-11-08T00:25:26,481 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/3bb2e114e9cf4bb2b7137cd736e336bf is 50, key is test_row_0/B:col10/1731025526031/Put/seqid=0 2024-11-08T00:25:26,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742292_1468 (size=12301) 2024-11-08T00:25:26,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:26,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025586741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:26,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:26,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025586743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:26,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:26,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:26,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:26,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:26,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:26,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:26,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:26,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025586744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:26,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:26,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025586744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:26,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:26,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:26,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:26,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:26,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:26,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:26,896 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/3bb2e114e9cf4bb2b7137cd736e336bf 2024-11-08T00:25:26,904 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/b2bf439f05eb427daf4cb9212751508b is 50, key is test_row_0/C:col10/1731025526031/Put/seqid=0 2024-11-08T00:25:26,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742293_1469 (size=12301) 2024-11-08T00:25:26,914 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/b2bf439f05eb427daf4cb9212751508b 2024-11-08T00:25:26,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/c5a9ebf28fe348389cd178d167fddb4b as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/c5a9ebf28fe348389cd178d167fddb4b 2024-11-08T00:25:26,925 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/c5a9ebf28fe348389cd178d167fddb4b, entries=250, sequenceid=364, filesize=47.5 K 2024-11-08T00:25:26,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/3bb2e114e9cf4bb2b7137cd736e336bf as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/3bb2e114e9cf4bb2b7137cd736e336bf 2024-11-08T00:25:26,930 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/3bb2e114e9cf4bb2b7137cd736e336bf, entries=150, sequenceid=364, filesize=12.0 K 2024-11-08T00:25:26,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/b2bf439f05eb427daf4cb9212751508b as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/b2bf439f05eb427daf4cb9212751508b 2024-11-08T00:25:26,934 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/b2bf439f05eb427daf4cb9212751508b, entries=150, sequenceid=364, filesize=12.0 K 2024-11-08T00:25:26,935 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 0d1e33fe1a366293d5e32ce198366168 in 903ms, sequenceid=364, compaction requested=true 2024-11-08T00:25:26,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0d1e33fe1a366293d5e32ce198366168: 2024-11-08T00:25:26,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d1e33fe1a366293d5e32ce198366168:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T00:25:26,935 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:25:26,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:25:26,935 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:25:26,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d1e33fe1a366293d5e32ce198366168:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T00:25:26,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:25:26,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d1e33fe1a366293d5e32ce198366168:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:25:26,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:25:26,936 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111899 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:25:26,936 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:25:26,936 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): 0d1e33fe1a366293d5e32ce198366168/B is initiating minor compaction (all files) 2024-11-08T00:25:26,936 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): 0d1e33fe1a366293d5e32ce198366168/A is initiating minor compaction (all files) 2024-11-08T00:25:26,936 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d1e33fe1a366293d5e32ce198366168/B in TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:26,936 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d1e33fe1a366293d5e32ce198366168/A in TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:26,936 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/321752b8ea214a9f96fdf1f6ff4f95a4, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/cb7b7d4d1a124c8eae05b690aab3d1b2, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/c5a9ebf28fe348389cd178d167fddb4b] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp, totalSize=109.3 K 2024-11-08T00:25:26,936 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/05ad2918b67b4840937dd29168b0f6f6, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/7ed8f60b900144088a51135fbb7ff9e2, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/3bb2e114e9cf4bb2b7137cd736e336bf] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp, totalSize=36.8 K 2024-11-08T00:25:26,936 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:26,936 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. files: [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/321752b8ea214a9f96fdf1f6ff4f95a4, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/cb7b7d4d1a124c8eae05b690aab3d1b2, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/c5a9ebf28fe348389cd178d167fddb4b] 2024-11-08T00:25:26,939 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 05ad2918b67b4840937dd29168b0f6f6, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731025523592 2024-11-08T00:25:26,939 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 321752b8ea214a9f96fdf1f6ff4f95a4, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731025523592 2024-11-08T00:25:26,939 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ed8f60b900144088a51135fbb7ff9e2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731025523656 2024-11-08T00:25:26,939 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb7b7d4d1a124c8eae05b690aab3d1b2, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731025523656 2024-11-08T00:25:26,941 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bb2e114e9cf4bb2b7137cd736e336bf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731025524865 2024-11-08T00:25:26,941 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5a9ebf28fe348389cd178d167fddb4b, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731025524865 2024-11-08T00:25:26,946 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0d1e33fe1a366293d5e32ce198366168] 2024-11-08T00:25:26,947 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d1e33fe1a366293d5e32ce198366168#B#compaction#395 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:25:26,948 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/2faf0ac124ef43579d6b4f3277922cad is 50, key is test_row_0/B:col10/1731025526031/Put/seqid=0 2024-11-08T00:25:26,948 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241108b725cde504b041cbb054e3483e69ea61_0d1e33fe1a366293d5e32ce198366168 store=[table=TestAcidGuarantees family=A region=0d1e33fe1a366293d5e32ce198366168] 2024-11-08T00:25:26,950 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241108b725cde504b041cbb054e3483e69ea61_0d1e33fe1a366293d5e32ce198366168, store=[table=TestAcidGuarantees family=A region=0d1e33fe1a366293d5e32ce198366168] 2024-11-08T00:25:26,950 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108b725cde504b041cbb054e3483e69ea61_0d1e33fe1a366293d5e32ce198366168 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0d1e33fe1a366293d5e32ce198366168] 2024-11-08T00:25:26,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742295_1471 (size=4469) 2024-11-08T00:25:26,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742294_1470 (size=13153) 2024-11-08T00:25:27,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:27,262 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:25:27,263 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0d1e33fe1a366293d5e32ce198366168 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-08T00:25:27,274 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108a5b30f7f4ae54c398040e126824e10a3_0d1e33fe1a366293d5e32ce198366168 is 50, key is test_row_0/A:col10/1731025527260/Put/seqid=0 2024-11-08T00:25:27,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742296_1472 (size=17534) 2024-11-08T00:25:27,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:27,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025587298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:27,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:27,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:27,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:27,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025587300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:27,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:27,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:27,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:27,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:27,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:27,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025587300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:27,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:27,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:27,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:27,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:27,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025587318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:27,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:27,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:27,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:27,359 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d1e33fe1a366293d5e32ce198366168#A#compaction#394 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:25:27,360 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/b5c1982a88124ab99cd0fa7b69c1b12a is 175, key is test_row_0/A:col10/1731025526031/Put/seqid=0 2024-11-08T00:25:27,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742297_1473 (size=32107) 2024-11-08T00:25:27,371 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/2faf0ac124ef43579d6b4f3277922cad as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/2faf0ac124ef43579d6b4f3277922cad 2024-11-08T00:25:27,376 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d1e33fe1a366293d5e32ce198366168/B of 0d1e33fe1a366293d5e32ce198366168 into 2faf0ac124ef43579d6b4f3277922cad(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:25:27,376 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d1e33fe1a366293d5e32ce198366168: 2024-11-08T00:25:27,376 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., storeName=0d1e33fe1a366293d5e32ce198366168/B, priority=13, startTime=1731025526935; duration=0sec 2024-11-08T00:25:27,376 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:25:27,376 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d1e33fe1a366293d5e32ce198366168:B 2024-11-08T00:25:27,376 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T00:25:27,377 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T00:25:27,377 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): 0d1e33fe1a366293d5e32ce198366168/C is initiating minor compaction (all files) 2024-11-08T00:25:27,377 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d1e33fe1a366293d5e32ce198366168/C in TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:27,377 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/01d8d4151e754493a4cd5dfce3c4a90f, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/91484d2347c64dad8fa0a8db38142a23, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/939518c2cffd4893a17d4ed07eb33787, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/b2bf439f05eb427daf4cb9212751508b] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp, totalSize=48.7 K 2024-11-08T00:25:27,377 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 01d8d4151e754493a4cd5dfce3c4a90f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1731025523081 2024-11-08T00:25:27,378 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 91484d2347c64dad8fa0a8db38142a23, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731025523592 2024-11-08T00:25:27,378 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 939518c2cffd4893a17d4ed07eb33787, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731025523656 2024-11-08T00:25:27,378 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting b2bf439f05eb427daf4cb9212751508b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731025524865 2024-11-08T00:25:27,385 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d1e33fe1a366293d5e32ce198366168#C#compaction#397 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:25:27,385 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/578701be7bd54befb29865d660bcf10c is 50, key is test_row_0/C:col10/1731025526031/Put/seqid=0 2024-11-08T00:25:27,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742298_1474 (size=13119) 2024-11-08T00:25:27,396 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/578701be7bd54befb29865d660bcf10c as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/578701be7bd54befb29865d660bcf10c 2024-11-08T00:25:27,399 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0d1e33fe1a366293d5e32ce198366168/C of 0d1e33fe1a366293d5e32ce198366168 into 578701be7bd54befb29865d660bcf10c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:25:27,399 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d1e33fe1a366293d5e32ce198366168: 2024-11-08T00:25:27,399 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., storeName=0d1e33fe1a366293d5e32ce198366168/C, priority=12, startTime=1731025526935; duration=0sec 2024-11-08T00:25:27,399 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:25:27,399 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d1e33fe1a366293d5e32ce198366168:C 2024-11-08T00:25:27,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:27,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025587427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:27,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:27,433 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:27,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025587427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:27,433 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:27,433 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:27,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:27,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:27,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:27,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:27,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025587428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:27,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:27,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:27,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:27,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:27,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025587437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:27,445 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:27,445 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:27,445 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:27,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-08T00:25:27,457 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:25:27,458 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-08T00:25:27,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-08T00:25:27,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-08T00:25:27,459 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T00:25:27,460 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T00:25:27,460 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T00:25:27,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-08T00:25:27,611 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-08T00:25:27,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:27,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. as already flushing 2024-11-08T00:25:27,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:27,612 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:27,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:27,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:27,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:27,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025587637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:27,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:27,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:27,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:27,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025587637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:27,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:27,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:27,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:27,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:27,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025587637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:27,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:27,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:27,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:27,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:27,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:27,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025587657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:27,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:27,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:27,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:27,681 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:27,683 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108a5b30f7f4ae54c398040e126824e10a3_0d1e33fe1a366293d5e32ce198366168 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108a5b30f7f4ae54c398040e126824e10a3_0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:27,684 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/929fae1f3c424403856c609296901bcf, store: [table=TestAcidGuarantees family=A region=0d1e33fe1a366293d5e32ce198366168] 2024-11-08T00:25:27,684 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/929fae1f3c424403856c609296901bcf is 175, key is test_row_0/A:col10/1731025527260/Put/seqid=0 2024-11-08T00:25:27,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742299_1475 (size=48639) 2024-11-08T00:25:27,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-08T00:25:27,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:27,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. as already flushing 2024-11-08T00:25:27,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:27,764 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:27,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:27,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:27,768 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/b5c1982a88124ab99cd0fa7b69c1b12a as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/b5c1982a88124ab99cd0fa7b69c1b12a 2024-11-08T00:25:27,772 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d1e33fe1a366293d5e32ce198366168/A of 0d1e33fe1a366293d5e32ce198366168 into b5c1982a88124ab99cd0fa7b69c1b12a(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:25:27,772 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d1e33fe1a366293d5e32ce198366168: 2024-11-08T00:25:27,772 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., storeName=0d1e33fe1a366293d5e32ce198366168/A, priority=13, startTime=1731025526935; duration=0sec 2024-11-08T00:25:27,773 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:25:27,773 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d1e33fe1a366293d5e32ce198366168:A 2024-11-08T00:25:27,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-08T00:25:27,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:27,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1731025587897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:27,903 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 3302f0f507bd,40479,1731025415856 for 'test_row_1' in 0d1e33fe1a366293d5e32ce198366168 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T00:25:27,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:27,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:27,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:27,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-08T00:25:27,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:27,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. as already flushing 2024-11-08T00:25:27,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:27,917 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:27,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:27,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:27,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:27,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025587957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:27,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:27,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:27,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:27,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:27,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025587957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:27,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:27,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:27,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:27,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:27,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025587957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:27,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:27,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:27,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:27,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:27,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025587968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:27,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:27,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:27,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:28,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-08T00:25:28,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:28,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. as already flushing 2024-11-08T00:25:28,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:28,070 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:28,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:28,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:28,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-08T00:25:28,088 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=387, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/929fae1f3c424403856c609296901bcf 2024-11-08T00:25:28,098 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/9cb114ebc8654ac88cf526b88a9571d0 is 50, key is test_row_0/B:col10/1731025527260/Put/seqid=0 2024-11-08T00:25:28,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742300_1476 (size=12301) 2024-11-08T00:25:28,106 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/9cb114ebc8654ac88cf526b88a9571d0 2024-11-08T00:25:28,114 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/81c651a7445e4107ae09e5aa8aa7f5b2 is 50, key is test_row_0/C:col10/1731025527260/Put/seqid=0 2024-11-08T00:25:28,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742301_1477 (size=12301) 2024-11-08T00:25:28,128 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/81c651a7445e4107ae09e5aa8aa7f5b2 2024-11-08T00:25:28,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/929fae1f3c424403856c609296901bcf as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/929fae1f3c424403856c609296901bcf 2024-11-08T00:25:28,141 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/929fae1f3c424403856c609296901bcf, entries=250, sequenceid=387, filesize=47.5 K 2024-11-08T00:25:28,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/9cb114ebc8654ac88cf526b88a9571d0 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/9cb114ebc8654ac88cf526b88a9571d0 2024-11-08T00:25:28,149 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/9cb114ebc8654ac88cf526b88a9571d0, entries=150, sequenceid=387, filesize=12.0 K 2024-11-08T00:25:28,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/81c651a7445e4107ae09e5aa8aa7f5b2 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/81c651a7445e4107ae09e5aa8aa7f5b2 2024-11-08T00:25:28,158 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/81c651a7445e4107ae09e5aa8aa7f5b2, entries=150, sequenceid=387, filesize=12.0 K 2024-11-08T00:25:28,159 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 0d1e33fe1a366293d5e32ce198366168 in 897ms, sequenceid=387, compaction requested=false 2024-11-08T00:25:28,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0d1e33fe1a366293d5e32ce198366168: 2024-11-08T00:25:28,223 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-08T00:25:28,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:28,224 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 0d1e33fe1a366293d5e32ce198366168 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-08T00:25:28,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108bd08789c74ad476f99eee08a8a92b5a0_0d1e33fe1a366293d5e32ce198366168 is 50, key is test_row_0/A:col10/1731025527295/Put/seqid=0 2024-11-08T00:25:28,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742302_1478 (size=12454) 2024-11-08T00:25:28,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:28,259 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108bd08789c74ad476f99eee08a8a92b5a0_0d1e33fe1a366293d5e32ce198366168 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108bd08789c74ad476f99eee08a8a92b5a0_0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:28,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/db633be2781b4b06993c353c548e49e4, store: [table=TestAcidGuarantees family=A region=0d1e33fe1a366293d5e32ce198366168] 2024-11-08T00:25:28,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/db633be2781b4b06993c353c548e49e4 is 175, key is test_row_0/A:col10/1731025527295/Put/seqid=0 2024-11-08T00:25:28,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742303_1479 (size=31255) 2024-11-08T00:25:28,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:28,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. as already flushing 2024-11-08T00:25:28,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:28,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025588552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:28,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:28,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:28,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:28,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:28,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025588554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:28,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:28,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025588555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:28,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:28,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:28,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:28,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:28,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:28,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:28,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:28,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025588564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:28,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:28,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:28,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:28,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-08T00:25:28,672 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=403, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/db633be2781b4b06993c353c548e49e4 2024-11-08T00:25:28,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:28,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025588667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:28,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/e590f4a3267c4d119195f366b6136bd2 is 50, key is test_row_0/B:col10/1731025527295/Put/seqid=0 2024-11-08T00:25:28,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:28,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:28,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:28,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:28,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025588677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:28,683 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:28,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:28,683 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:28,683 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:28,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025588677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:28,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:28,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:28,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:28,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742304_1480 (size=12301) 2024-11-08T00:25:28,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:28,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025588687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:28,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:28,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:28,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:28,695 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/e590f4a3267c4d119195f366b6136bd2 2024-11-08T00:25:28,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/a2982a612631460c94350015ba1a984a is 50, key is test_row_0/C:col10/1731025527295/Put/seqid=0 2024-11-08T00:25:28,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742305_1481 (size=12301) 2024-11-08T00:25:28,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:28,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:28,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:28,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025588887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:28,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025588887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:28,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025588887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:28,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:28,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025588897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:28,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:28,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:28,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:28,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:28,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:28,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:28,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:28,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:28,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:28,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:28,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:28,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:29,118 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/a2982a612631460c94350015ba1a984a 2024-11-08T00:25:29,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/db633be2781b4b06993c353c548e49e4 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/db633be2781b4b06993c353c548e49e4 2024-11-08T00:25:29,125 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/db633be2781b4b06993c353c548e49e4, entries=150, sequenceid=403, filesize=30.5 K 2024-11-08T00:25:29,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/e590f4a3267c4d119195f366b6136bd2 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/e590f4a3267c4d119195f366b6136bd2 2024-11-08T00:25:29,130 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/e590f4a3267c4d119195f366b6136bd2, entries=150, sequenceid=403, filesize=12.0 K 2024-11-08T00:25:29,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/a2982a612631460c94350015ba1a984a as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/a2982a612631460c94350015ba1a984a 2024-11-08T00:25:29,135 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/a2982a612631460c94350015ba1a984a, entries=150, sequenceid=403, filesize=12.0 K 2024-11-08T00:25:29,136 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 0d1e33fe1a366293d5e32ce198366168 in 913ms, sequenceid=403, compaction requested=true 2024-11-08T00:25:29,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 0d1e33fe1a366293d5e32ce198366168: 2024-11-08T00:25:29,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:29,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-08T00:25:29,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-08T00:25:29,139 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-08T00:25:29,140 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6770 sec 2024-11-08T00:25:29,142 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.6820 sec 2024-11-08T00:25:29,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:29,215 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:25:29,215 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0d1e33fe1a366293d5e32ce198366168 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-08T00:25:29,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108c73ca7db72d942d2a33f6556dc25012e_0d1e33fe1a366293d5e32ce198366168 is 50, key is test_row_0/A:col10/1731025528560/Put/seqid=0 2024-11-08T00:25:29,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742306_1482 (size=17534) 2024-11-08T00:25:29,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:29,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025589232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:29,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:29,249 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:29,249 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:29,249 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:29,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025589233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:29,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:29,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:29,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:29,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:29,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025589234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:29,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:29,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:29,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:29,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:29,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025589239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:29,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:29,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:29,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:29,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:29,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025589357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:29,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:29,367 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:29,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025589357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:29,367 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:29,367 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:29,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:29,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:29,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:29,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:29,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025589357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:29,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:29,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:29,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:29,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:29,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025589357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:29,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:29,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:29,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:29,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:29,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025589577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:29,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:29,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:29,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025589577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:29,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:29,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:29,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:29,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:29,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:29,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:29,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025589577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:29,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:29,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:29,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:29,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:29,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025589577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:29,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:29,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:29,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:29,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-08T00:25:29,607 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:25:29,608 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-08T00:25:29,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-08T00:25:29,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-08T00:25:29,610 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T00:25:29,611 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T00:25:29,611 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T00:25:29,632 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:29,636 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108c73ca7db72d942d2a33f6556dc25012e_0d1e33fe1a366293d5e32ce198366168 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108c73ca7db72d942d2a33f6556dc25012e_0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:29,637 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/ad3750a047d4495eba007673f2259cc4, store: [table=TestAcidGuarantees family=A region=0d1e33fe1a366293d5e32ce198366168] 2024-11-08T00:25:29,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/ad3750a047d4495eba007673f2259cc4 is 175, key is test_row_0/A:col10/1731025528560/Put/seqid=0 2024-11-08T00:25:29,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742307_1483 (size=48639) 2024-11-08T00:25:29,643 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=428, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/ad3750a047d4495eba007673f2259cc4 2024-11-08T00:25:29,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/346bf136876544df852b906d17971f76 is 50, key is test_row_0/B:col10/1731025528560/Put/seqid=0 2024-11-08T00:25:29,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742308_1484 (size=12301) 2024-11-08T00:25:29,659 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/346bf136876544df852b906d17971f76 2024-11-08T00:25:29,671 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/ac23323a26ce4b45889f84699f96db84 is 50, key is test_row_0/C:col10/1731025528560/Put/seqid=0 2024-11-08T00:25:29,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742309_1485 (size=12301) 2024-11-08T00:25:29,684 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/ac23323a26ce4b45889f84699f96db84 2024-11-08T00:25:29,688 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/ad3750a047d4495eba007673f2259cc4 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/ad3750a047d4495eba007673f2259cc4 2024-11-08T00:25:29,696 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/ad3750a047d4495eba007673f2259cc4, entries=250, sequenceid=428, filesize=47.5 K 2024-11-08T00:25:29,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/346bf136876544df852b906d17971f76 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/346bf136876544df852b906d17971f76 2024-11-08T00:25:29,700 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/346bf136876544df852b906d17971f76, entries=150, sequenceid=428, filesize=12.0 K 2024-11-08T00:25:29,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/ac23323a26ce4b45889f84699f96db84 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/ac23323a26ce4b45889f84699f96db84 2024-11-08T00:25:29,705 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/ac23323a26ce4b45889f84699f96db84, entries=150, sequenceid=428, filesize=12.0 K 2024-11-08T00:25:29,706 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 0d1e33fe1a366293d5e32ce198366168 in 491ms, sequenceid=428, compaction requested=true 2024-11-08T00:25:29,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0d1e33fe1a366293d5e32ce198366168: 2024-11-08T00:25:29,706 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T00:25:29,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d1e33fe1a366293d5e32ce198366168:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T00:25:29,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:25:29,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d1e33fe1a366293d5e32ce198366168:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T00:25:29,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:25:29,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d1e33fe1a366293d5e32ce198366168:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:25:29,707 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T00:25:29,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:25:29,711 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 160640 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T00:25:29,711 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): 0d1e33fe1a366293d5e32ce198366168/A is initiating minor compaction (all files) 2024-11-08T00:25:29,711 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d1e33fe1a366293d5e32ce198366168/A in TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:29,711 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/b5c1982a88124ab99cd0fa7b69c1b12a, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/929fae1f3c424403856c609296901bcf, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/db633be2781b4b06993c353c548e49e4, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/ad3750a047d4495eba007673f2259cc4] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp, totalSize=156.9 K 2024-11-08T00:25:29,711 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:29,711 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. files: [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/b5c1982a88124ab99cd0fa7b69c1b12a, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/929fae1f3c424403856c609296901bcf, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/db633be2781b4b06993c353c548e49e4, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/ad3750a047d4495eba007673f2259cc4] 2024-11-08T00:25:29,712 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T00:25:29,712 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5c1982a88124ab99cd0fa7b69c1b12a, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731025524865 2024-11-08T00:25:29,712 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): 0d1e33fe1a366293d5e32ce198366168/B is initiating minor compaction (all files) 2024-11-08T00:25:29,712 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d1e33fe1a366293d5e32ce198366168/B in TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:29,712 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/2faf0ac124ef43579d6b4f3277922cad, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/9cb114ebc8654ac88cf526b88a9571d0, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/e590f4a3267c4d119195f366b6136bd2, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/346bf136876544df852b906d17971f76] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp, totalSize=48.9 K 2024-11-08T00:25:29,712 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 929fae1f3c424403856c609296901bcf, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1731025526083 2024-11-08T00:25:29,712 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 2faf0ac124ef43579d6b4f3277922cad, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731025524865 2024-11-08T00:25:29,713 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting db633be2781b4b06993c353c548e49e4, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1731025527274 2024-11-08T00:25:29,713 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 9cb114ebc8654ac88cf526b88a9571d0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1731025526083 2024-11-08T00:25:29,713 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad3750a047d4495eba007673f2259cc4, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1731025528545 2024-11-08T00:25:29,713 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting e590f4a3267c4d119195f366b6136bd2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1731025527274 2024-11-08T00:25:29,714 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 346bf136876544df852b906d17971f76, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1731025528552 2024-11-08T00:25:29,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-08T00:25:29,729 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0d1e33fe1a366293d5e32ce198366168] 2024-11-08T00:25:29,737 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d1e33fe1a366293d5e32ce198366168#B#compaction#407 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:25:29,738 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/ad5d4dfadef04907b3ce368e4460ee9c is 50, key is test_row_0/B:col10/1731025528560/Put/seqid=0 2024-11-08T00:25:29,744 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241108e068ae5369de4f6ba5fea50b959a4118_0d1e33fe1a366293d5e32ce198366168 store=[table=TestAcidGuarantees family=A region=0d1e33fe1a366293d5e32ce198366168] 2024-11-08T00:25:29,746 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241108e068ae5369de4f6ba5fea50b959a4118_0d1e33fe1a366293d5e32ce198366168, store=[table=TestAcidGuarantees family=A region=0d1e33fe1a366293d5e32ce198366168] 2024-11-08T00:25:29,747 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108e068ae5369de4f6ba5fea50b959a4118_0d1e33fe1a366293d5e32ce198366168 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0d1e33fe1a366293d5e32ce198366168] 2024-11-08T00:25:29,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742310_1486 (size=13289) 2024-11-08T00:25:29,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742311_1487 (size=4469) 2024-11-08T00:25:29,760 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d1e33fe1a366293d5e32ce198366168#A#compaction#406 average throughput is 0.79 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:25:29,761 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/15367e8586e34407b8097f734c1bf8aa is 175, key is test_row_0/A:col10/1731025528560/Put/seqid=0 2024-11-08T00:25:29,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-08T00:25:29,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:29,765 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 0d1e33fe1a366293d5e32ce198366168 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T00:25:29,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108548329b68be8417597893de2609d3d03_0d1e33fe1a366293d5e32ce198366168 is 50, key is test_row_0/A:col10/1731025529237/Put/seqid=0 2024-11-08T00:25:29,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742312_1488 (size=32243) 2024-11-08T00:25:29,787 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/15367e8586e34407b8097f734c1bf8aa as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/15367e8586e34407b8097f734c1bf8aa 2024-11-08T00:25:29,792 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0d1e33fe1a366293d5e32ce198366168/A of 0d1e33fe1a366293d5e32ce198366168 into 15367e8586e34407b8097f734c1bf8aa(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:25:29,792 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d1e33fe1a366293d5e32ce198366168: 2024-11-08T00:25:29,792 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., storeName=0d1e33fe1a366293d5e32ce198366168/A, priority=12, startTime=1731025529706; duration=0sec 2024-11-08T00:25:29,792 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:25:29,792 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d1e33fe1a366293d5e32ce198366168:A 2024-11-08T00:25:29,792 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T00:25:29,793 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T00:25:29,794 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): 0d1e33fe1a366293d5e32ce198366168/C is initiating minor compaction (all files) 2024-11-08T00:25:29,794 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d1e33fe1a366293d5e32ce198366168/C in TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:29,794 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/578701be7bd54befb29865d660bcf10c, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/81c651a7445e4107ae09e5aa8aa7f5b2, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/a2982a612631460c94350015ba1a984a, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/ac23323a26ce4b45889f84699f96db84] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp, totalSize=48.8 K 2024-11-08T00:25:29,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742313_1489 (size=12454) 2024-11-08T00:25:29,794 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 578701be7bd54befb29865d660bcf10c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731025524865 2024-11-08T00:25:29,795 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81c651a7445e4107ae09e5aa8aa7f5b2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1731025526083 2024-11-08T00:25:29,795 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2982a612631460c94350015ba1a984a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1731025527274 2024-11-08T00:25:29,796 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac23323a26ce4b45889f84699f96db84, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1731025528552 2024-11-08T00:25:29,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:29,803 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108548329b68be8417597893de2609d3d03_0d1e33fe1a366293d5e32ce198366168 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108548329b68be8417597893de2609d3d03_0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:29,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/0d34644c817e4b1b911c193125ce84b8, store: [table=TestAcidGuarantees family=A region=0d1e33fe1a366293d5e32ce198366168] 2024-11-08T00:25:29,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/0d34644c817e4b1b911c193125ce84b8 is 175, key is test_row_0/A:col10/1731025529237/Put/seqid=0 2024-11-08T00:25:29,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742314_1490 (size=31255) 2024-11-08T00:25:29,818 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=439, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/0d34644c817e4b1b911c193125ce84b8 2024-11-08T00:25:29,823 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d1e33fe1a366293d5e32ce198366168#C#compaction#409 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:25:29,823 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/f7fd1ced3afb4605882855ca99eed005 is 50, key is test_row_0/C:col10/1731025528560/Put/seqid=0 2024-11-08T00:25:29,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/cf373299764b423d871ea331787b7f0d is 50, key is test_row_0/B:col10/1731025529237/Put/seqid=0 2024-11-08T00:25:29,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742315_1491 (size=13255) 2024-11-08T00:25:29,856 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/f7fd1ced3afb4605882855ca99eed005 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/f7fd1ced3afb4605882855ca99eed005 2024-11-08T00:25:29,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742316_1492 (size=12301) 2024-11-08T00:25:29,864 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0d1e33fe1a366293d5e32ce198366168/C of 0d1e33fe1a366293d5e32ce198366168 into f7fd1ced3afb4605882855ca99eed005(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:25:29,864 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d1e33fe1a366293d5e32ce198366168: 2024-11-08T00:25:29,864 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., storeName=0d1e33fe1a366293d5e32ce198366168/C, priority=12, startTime=1731025529706; duration=0sec 2024-11-08T00:25:29,864 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:25:29,864 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d1e33fe1a366293d5e32ce198366168:C 2024-11-08T00:25:29,867 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/cf373299764b423d871ea331787b7f0d 2024-11-08T00:25:29,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/0fcf5e87825b46c1b833e134a1bd23cb is 50, key is test_row_0/C:col10/1731025529237/Put/seqid=0 2024-11-08T00:25:29,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742317_1493 (size=12301) 2024-11-08T00:25:29,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:29,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. as already flushing 2024-11-08T00:25:29,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-08T00:25:29,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:29,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025589955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:29,959 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:29,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:29,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:29,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:29,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025589957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:29,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:29,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:29,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:29,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:29,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025589959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:29,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:29,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:29,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:29,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025589959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:29,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:29,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:29,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:29,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:30,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:30,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025590067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:30,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:30,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:30,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:30,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:30,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025590067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:30,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:30,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:30,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:30,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:30,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025590077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:30,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:30,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025590077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:30,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:30,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:30,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:30,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:30,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:30,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:30,155 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/ad5d4dfadef04907b3ce368e4460ee9c as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/ad5d4dfadef04907b3ce368e4460ee9c 2024-11-08T00:25:30,160 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0d1e33fe1a366293d5e32ce198366168/B of 0d1e33fe1a366293d5e32ce198366168 into ad5d4dfadef04907b3ce368e4460ee9c(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:25:30,160 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d1e33fe1a366293d5e32ce198366168: 2024-11-08T00:25:30,160 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., storeName=0d1e33fe1a366293d5e32ce198366168/B, priority=12, startTime=1731025529706; duration=0sec 2024-11-08T00:25:30,160 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:25:30,160 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d1e33fe1a366293d5e32ce198366168:B 2024-11-08T00:25:30,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-08T00:25:30,288 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/0fcf5e87825b46c1b833e134a1bd23cb 2024-11-08T00:25:30,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/0d34644c817e4b1b911c193125ce84b8 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/0d34644c817e4b1b911c193125ce84b8 2024-11-08T00:25:30,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:30,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025590287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:30,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:30,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025590287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:30,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:30,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025590287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:30,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:30,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025590288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:30,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:30,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:30,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:30,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:30,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:30,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:30,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:30,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:30,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:30,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:30,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:30,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:30,304 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/0d34644c817e4b1b911c193125ce84b8, entries=150, sequenceid=439, filesize=30.5 K 2024-11-08T00:25:30,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/cf373299764b423d871ea331787b7f0d as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/cf373299764b423d871ea331787b7f0d 2024-11-08T00:25:30,308 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/cf373299764b423d871ea331787b7f0d, entries=150, sequenceid=439, filesize=12.0 K 2024-11-08T00:25:30,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/0fcf5e87825b46c1b833e134a1bd23cb as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/0fcf5e87825b46c1b833e134a1bd23cb 2024-11-08T00:25:30,314 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/0fcf5e87825b46c1b833e134a1bd23cb, entries=150, sequenceid=439, filesize=12.0 K 2024-11-08T00:25:30,315 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 0d1e33fe1a366293d5e32ce198366168 in 550ms, sequenceid=439, compaction requested=false 2024-11-08T00:25:30,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 0d1e33fe1a366293d5e32ce198366168: 2024-11-08T00:25:30,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:30,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-08T00:25:30,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-08T00:25:30,318 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-08T00:25:30,318 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 705 msec 2024-11-08T00:25:30,320 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 711 msec 2024-11-08T00:25:30,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:30,611 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:25:30,611 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0d1e33fe1a366293d5e32ce198366168 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-08T00:25:30,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411088e35ecd5cc60442c99b4cd33c913d583_0d1e33fe1a366293d5e32ce198366168 is 50, key is test_row_0/A:col10/1731025529955/Put/seqid=0 2024-11-08T00:25:30,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742318_1494 (size=14994) 2024-11-08T00:25:30,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:30,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025590624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:30,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:30,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:30,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:30,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:30,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025590626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:30,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:30,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025590626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:30,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:30,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:30,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:30,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:30,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025590627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:30,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:30,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:30,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:30,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:30,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:30,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:30,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:30,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025590738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:30,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:30,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:30,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:30,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:30,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025590738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:30,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:30,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:30,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:30,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:30,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025590738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:30,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:30,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:30,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:30,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:30,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025590738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:30,745 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:30,745 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:30,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:30,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-08T00:25:30,747 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:25:30,747 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-08T00:25:30,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-08T00:25:30,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-08T00:25:30,749 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T00:25:30,750 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T00:25:30,750 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T00:25:30,765 INFO [Thread-1685 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1685. 2024-11-08T00:25:30,765 DEBUG [Thread-1685 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T00:25:30,765 DEBUG [Thread-1685 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:30,765 INFO [Thread-1689 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1689. 2024-11-08T00:25:30,765 DEBUG [Thread-1689 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T00:25:30,765 DEBUG [Thread-1685 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:30,765 DEBUG [Thread-1689 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:30,765 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:25:30,766 DEBUG [Thread-1689 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:30,766 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:25:30,767 INFO [Thread-1683 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1683. 2024-11-08T00:25:30,768 DEBUG [Thread-1683 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T00:25:30,768 DEBUG [Thread-1683 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:30,768 DEBUG [Thread-1683 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:30,768 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:25:30,773 INFO [Thread-1687 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1687. 2024-11-08T00:25:30,773 DEBUG [Thread-1687 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T00:25:30,773 DEBUG [Thread-1687 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:30,774 DEBUG [Thread-1687 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:30,774 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:25:30,774 INFO [Thread-1681 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1681. 2024-11-08T00:25:30,774 DEBUG [Thread-1681 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T00:25:30,774 DEBUG [Thread-1681 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:30,774 DEBUG [Thread-1681 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:30,774 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:25:30,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-08T00:25:30,902 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-08T00:25:30,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:30,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. as already flushing 2024-11-08T00:25:30,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:30,903 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:30,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:30,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:30,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:30,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:30,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025590947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:30,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025590947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:30,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:30,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:30,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:30,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:30,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:30,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:30,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:30,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:30,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025590957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:30,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025590957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:30,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:30,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:30,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:30,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:30,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:30,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:31,022 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:31,025 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411088e35ecd5cc60442c99b4cd33c913d583_0d1e33fe1a366293d5e32ce198366168 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411088e35ecd5cc60442c99b4cd33c913d583_0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:31,026 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/654017f7ceeb45ab81c634cae286219c, store: [table=TestAcidGuarantees family=A region=0d1e33fe1a366293d5e32ce198366168] 2024-11-08T00:25:31,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/654017f7ceeb45ab81c634cae286219c is 175, key is test_row_0/A:col10/1731025529955/Put/seqid=0 2024-11-08T00:25:31,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742319_1495 (size=39949) 2024-11-08T00:25:31,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-08T00:25:31,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:31,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. as already flushing 2024-11-08T00:25:31,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:31,056 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:31,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:31,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:31,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-08T00:25:31,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-08T00:25:31,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:31,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. as already flushing 2024-11-08T00:25:31,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:31,209 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:31,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:31,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:31,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:31,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:31,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025591257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:31,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025591257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:31,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:31,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:31,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:31,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:31,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:31,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:31,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:31,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:31,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025591267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:31,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025591268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:31,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:31,268 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:31,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:31,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:31,268 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:31,268 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:31,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-08T00:25:31,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:31,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. as already flushing 2024-11-08T00:25:31,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:31,362 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:31,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:31,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:31,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-08T00:25:31,430 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=468, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/654017f7ceeb45ab81c634cae286219c 2024-11-08T00:25:31,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/0c7055d4f1654f23b73afe87b683eca6 is 50, key is test_row_0/B:col10/1731025529955/Put/seqid=0 2024-11-08T00:25:31,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742320_1496 (size=12301) 2024-11-08T00:25:31,515 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-08T00:25:31,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:31,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. as already flushing 2024-11-08T00:25:31,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:31,515 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:31,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:31,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:31,668 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-08T00:25:31,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:31,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. as already flushing 2024-11-08T00:25:31,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:31,668 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:31,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:31,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:31,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:31,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:31,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47250 deadline: 1731025591767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:31,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47256 deadline: 1731025591767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:31,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:31,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:31,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:31,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:31,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:31,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:31,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:31,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:31,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47272 deadline: 1731025591777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:31,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47200 deadline: 1731025591777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:31,778 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:31,778 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:31,778 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:31,778 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:31,778 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:31,778 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:31,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-08T00:25:31,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:31,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. as already flushing 2024-11-08T00:25:31,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:31,821 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:31,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:31,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:31,840 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/0c7055d4f1654f23b73afe87b683eca6 2024-11-08T00:25:31,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/8651f233a1334553929c686943db49f0 is 50, key is test_row_0/C:col10/1731025529955/Put/seqid=0 2024-11-08T00:25:31,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742321_1497 (size=12301) 2024-11-08T00:25:31,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-08T00:25:31,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:31,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47238 deadline: 1731025591917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:31,918 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 3302f0f507bd,40479,1731025415856 for 'test_row_1' in 0d1e33fe1a366293d5e32ce198366168 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8238 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T00:25:31,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:31,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:31,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:25:31,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-08T00:25:31,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:31,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. as already flushing 2024-11-08T00:25:31,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:31,974 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:31,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:31,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:32,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-08T00:25:32,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:32,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. as already flushing 2024-11-08T00:25:32,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:32,127 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:32,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:32,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:32,250 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/8651f233a1334553929c686943db49f0 2024-11-08T00:25:32,280 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-08T00:25:32,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:32,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. as already flushing 2024-11-08T00:25:32,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:32,280 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:32,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:32,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:32,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/654017f7ceeb45ab81c634cae286219c as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/654017f7ceeb45ab81c634cae286219c 2024-11-08T00:25:32,307 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/654017f7ceeb45ab81c634cae286219c, entries=200, sequenceid=468, filesize=39.0 K 2024-11-08T00:25:32,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/0c7055d4f1654f23b73afe87b683eca6 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/0c7055d4f1654f23b73afe87b683eca6 2024-11-08T00:25:32,311 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/0c7055d4f1654f23b73afe87b683eca6, entries=150, sequenceid=468, filesize=12.0 K 2024-11-08T00:25:32,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/8651f233a1334553929c686943db49f0 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/8651f233a1334553929c686943db49f0 2024-11-08T00:25:32,314 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/8651f233a1334553929c686943db49f0, entries=150, sequenceid=468, filesize=12.0 K 2024-11-08T00:25:32,315 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 0d1e33fe1a366293d5e32ce198366168 in 1704ms, sequenceid=468, compaction requested=true 2024-11-08T00:25:32,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0d1e33fe1a366293d5e32ce198366168: 2024-11-08T00:25:32,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d1e33fe1a366293d5e32ce198366168:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T00:25:32,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:25:32,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d1e33fe1a366293d5e32ce198366168:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T00:25:32,315 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:25:32,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:25:32,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0d1e33fe1a366293d5e32ce198366168:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:25:32,315 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:25:32,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:25:32,316 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103447 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:25:32,316 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:25:32,316 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): 0d1e33fe1a366293d5e32ce198366168/B is initiating minor compaction (all files) 2024-11-08T00:25:32,316 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): 0d1e33fe1a366293d5e32ce198366168/A is initiating minor compaction (all files) 2024-11-08T00:25:32,316 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d1e33fe1a366293d5e32ce198366168/A in TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:32,316 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d1e33fe1a366293d5e32ce198366168/B in TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:32,316 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/ad5d4dfadef04907b3ce368e4460ee9c, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/cf373299764b423d871ea331787b7f0d, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/0c7055d4f1654f23b73afe87b683eca6] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp, totalSize=37.0 K 2024-11-08T00:25:32,316 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/15367e8586e34407b8097f734c1bf8aa, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/0d34644c817e4b1b911c193125ce84b8, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/654017f7ceeb45ab81c634cae286219c] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp, totalSize=101.0 K 2024-11-08T00:25:32,316 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:32,316 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. files: [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/15367e8586e34407b8097f734c1bf8aa, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/0d34644c817e4b1b911c193125ce84b8, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/654017f7ceeb45ab81c634cae286219c] 2024-11-08T00:25:32,316 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting ad5d4dfadef04907b3ce368e4460ee9c, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1731025528552 2024-11-08T00:25:32,316 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15367e8586e34407b8097f734c1bf8aa, keycount=150, bloomtype=ROW, size=31.5 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1731025528552 2024-11-08T00:25:32,316 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting cf373299764b423d871ea331787b7f0d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1731025529223 2024-11-08T00:25:32,316 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d34644c817e4b1b911c193125ce84b8, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1731025529223 2024-11-08T00:25:32,317 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c7055d4f1654f23b73afe87b683eca6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1731025529955 2024-11-08T00:25:32,317 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 654017f7ceeb45ab81c634cae286219c, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1731025529955 2024-11-08T00:25:32,321 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0d1e33fe1a366293d5e32ce198366168] 2024-11-08T00:25:32,322 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d1e33fe1a366293d5e32ce198366168#B#compaction#415 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:25:32,322 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241108cca9b71a4e9646bca9df6278ea774ba8_0d1e33fe1a366293d5e32ce198366168 store=[table=TestAcidGuarantees family=A region=0d1e33fe1a366293d5e32ce198366168] 2024-11-08T00:25:32,322 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/11de1e334de24ffeb7d1b33360cddd0b is 50, key is test_row_0/B:col10/1731025529955/Put/seqid=0 2024-11-08T00:25:32,326 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241108cca9b71a4e9646bca9df6278ea774ba8_0d1e33fe1a366293d5e32ce198366168, store=[table=TestAcidGuarantees family=A region=0d1e33fe1a366293d5e32ce198366168] 2024-11-08T00:25:32,326 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108cca9b71a4e9646bca9df6278ea774ba8_0d1e33fe1a366293d5e32ce198366168 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0d1e33fe1a366293d5e32ce198366168] 2024-11-08T00:25:32,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742322_1498 (size=13391) 2024-11-08T00:25:32,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742323_1499 (size=4469) 2024-11-08T00:25:32,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-08T00:25:32,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:32,453 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 0d1e33fe1a366293d5e32ce198366168 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-08T00:25:32,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411085dc0f4aa26594acabf971b1f4d4d8f37_0d1e33fe1a366293d5e32ce198366168 is 50, key is test_row_0/A:col10/1731025530625/Put/seqid=0 2024-11-08T00:25:32,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742324_1500 (size=12454) 2024-11-08T00:25:32,730 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d1e33fe1a366293d5e32ce198366168#A#compaction#416 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:25:32,731 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/56e2138691914096892edd06bd8e4c26 is 175, key is test_row_0/A:col10/1731025529955/Put/seqid=0 2024-11-08T00:25:32,733 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/11de1e334de24ffeb7d1b33360cddd0b as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/11de1e334de24ffeb7d1b33360cddd0b 2024-11-08T00:25:32,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742325_1501 (size=32345) 2024-11-08T00:25:32,738 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d1e33fe1a366293d5e32ce198366168/B of 0d1e33fe1a366293d5e32ce198366168 into 11de1e334de24ffeb7d1b33360cddd0b(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:25:32,738 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d1e33fe1a366293d5e32ce198366168: 2024-11-08T00:25:32,738 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., storeName=0d1e33fe1a366293d5e32ce198366168/B, priority=13, startTime=1731025532315; duration=0sec 2024-11-08T00:25:32,738 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:25:32,738 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d1e33fe1a366293d5e32ce198366168:B 2024-11-08T00:25:32,738 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:25:32,739 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:25:32,739 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): 0d1e33fe1a366293d5e32ce198366168/C is initiating minor compaction (all files) 2024-11-08T00:25:32,739 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0d1e33fe1a366293d5e32ce198366168/C in TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:32,739 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/f7fd1ced3afb4605882855ca99eed005, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/0fcf5e87825b46c1b833e134a1bd23cb, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/8651f233a1334553929c686943db49f0] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp, totalSize=37.0 K 2024-11-08T00:25:32,740 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting f7fd1ced3afb4605882855ca99eed005, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1731025528552 2024-11-08T00:25:32,740 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 0fcf5e87825b46c1b833e134a1bd23cb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1731025529223 2024-11-08T00:25:32,740 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 8651f233a1334553929c686943db49f0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1731025529955 2024-11-08T00:25:32,746 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0d1e33fe1a366293d5e32ce198366168#C#compaction#418 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:25:32,746 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/7c131c56123c4430bc4b587eabcd34e6 is 50, key is test_row_0/C:col10/1731025529955/Put/seqid=0 2024-11-08T00:25:32,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742326_1502 (size=13357) 2024-11-08T00:25:32,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:32,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. as already flushing 2024-11-08T00:25:32,779 INFO [Thread-1676 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1676. 2024-11-08T00:25:32,779 DEBUG [Thread-1676 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T00:25:32,779 DEBUG [Thread-1676 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:32,779 DEBUG [Thread-1676 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:32,779 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:25:32,788 INFO [Thread-1670 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1670. 2024-11-08T00:25:32,788 INFO [Thread-1674 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1674. 2024-11-08T00:25:32,788 DEBUG [Thread-1670 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T00:25:32,788 DEBUG [Thread-1674 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T00:25:32,788 DEBUG [Thread-1670 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:32,788 DEBUG [Thread-1674 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:32,788 DEBUG [Thread-1674 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:32,788 DEBUG [Thread-1670 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:32,788 INFO [Thread-1672 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1672. 2024-11-08T00:25:32,788 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:25:32,788 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:25:32,788 DEBUG [Thread-1672 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T00:25:32,788 DEBUG [Thread-1672 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:32,789 DEBUG [Thread-1672 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:32,789 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:25:32,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:32,868 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411085dc0f4aa26594acabf971b1f4d4d8f37_0d1e33fe1a366293d5e32ce198366168 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411085dc0f4aa26594acabf971b1f4d4d8f37_0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:32,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/e9472ac2f11c473e946b8689999abb7f, store: [table=TestAcidGuarantees family=A region=0d1e33fe1a366293d5e32ce198366168] 2024-11-08T00:25:32,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/e9472ac2f11c473e946b8689999abb7f is 175, key is test_row_0/A:col10/1731025530625/Put/seqid=0 2024-11-08T00:25:32,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742327_1503 (size=31255) 2024-11-08T00:25:32,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-08T00:25:32,918 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T00:25:33,139 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/56e2138691914096892edd06bd8e4c26 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/56e2138691914096892edd06bd8e4c26 2024-11-08T00:25:33,143 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d1e33fe1a366293d5e32ce198366168/A of 0d1e33fe1a366293d5e32ce198366168 into 56e2138691914096892edd06bd8e4c26(size=31.6 K), total size for store is 31.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:25:33,143 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d1e33fe1a366293d5e32ce198366168: 2024-11-08T00:25:33,143 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., storeName=0d1e33fe1a366293d5e32ce198366168/A, priority=13, startTime=1731025532315; duration=0sec 2024-11-08T00:25:33,143 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:25:33,143 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d1e33fe1a366293d5e32ce198366168:A 2024-11-08T00:25:33,153 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/7c131c56123c4430bc4b587eabcd34e6 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/7c131c56123c4430bc4b587eabcd34e6 2024-11-08T00:25:33,156 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0d1e33fe1a366293d5e32ce198366168/C of 0d1e33fe1a366293d5e32ce198366168 into 7c131c56123c4430bc4b587eabcd34e6(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:25:33,156 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0d1e33fe1a366293d5e32ce198366168: 2024-11-08T00:25:33,156 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., storeName=0d1e33fe1a366293d5e32ce198366168/C, priority=13, startTime=1731025532315; duration=0sec 2024-11-08T00:25:33,157 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:25:33,157 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0d1e33fe1a366293d5e32ce198366168:C 2024-11-08T00:25:33,273 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=478, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/e9472ac2f11c473e946b8689999abb7f 2024-11-08T00:25:33,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/4ba5c337c2c0456ba935272998884427 is 50, key is test_row_0/B:col10/1731025530625/Put/seqid=0 2024-11-08T00:25:33,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742328_1504 (size=12301) 2024-11-08T00:25:33,686 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/4ba5c337c2c0456ba935272998884427 2024-11-08T00:25:33,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/42869447eade4dd9899fc8f74f324e3e is 50, key is test_row_0/C:col10/1731025530625/Put/seqid=0 2024-11-08T00:25:33,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742329_1505 (size=12301) 2024-11-08T00:25:34,096 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/42869447eade4dd9899fc8f74f324e3e 2024-11-08T00:25:34,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/e9472ac2f11c473e946b8689999abb7f as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/e9472ac2f11c473e946b8689999abb7f 2024-11-08T00:25:34,103 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/e9472ac2f11c473e946b8689999abb7f, entries=150, sequenceid=478, filesize=30.5 K 2024-11-08T00:25:34,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/4ba5c337c2c0456ba935272998884427 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/4ba5c337c2c0456ba935272998884427 2024-11-08T00:25:34,107 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/4ba5c337c2c0456ba935272998884427, entries=150, sequenceid=478, filesize=12.0 K 2024-11-08T00:25:34,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/42869447eade4dd9899fc8f74f324e3e as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/42869447eade4dd9899fc8f74f324e3e 2024-11-08T00:25:34,111 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/42869447eade4dd9899fc8f74f324e3e, entries=150, sequenceid=478, filesize=12.0 K 2024-11-08T00:25:34,112 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=26.84 KB/27480 for 0d1e33fe1a366293d5e32ce198366168 in 1659ms, sequenceid=478, compaction requested=false 2024-11-08T00:25:34,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 0d1e33fe1a366293d5e32ce198366168: 2024-11-08T00:25:34,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:34,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-08T00:25:34,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-08T00:25:34,115 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-08T00:25:34,115 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3630 sec 2024-11-08T00:25:34,117 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 3.3670 sec 2024-11-08T00:25:34,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-08T00:25:34,907 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:25:39,628 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/3ade3e156ded429a8d3306068b0912fa, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/9001e2bc424d493399048645ccc55b6c, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/abded7b318194d89a9e4ff8268fa1d1f, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/b9a34facbd464d91bebd049fa5c8fc17, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/947c6958e4ca4dec97be3f4961bdc951, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/8237934a842645a1a175adabf738c223, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/3d637170ec014ab58f0cc2b63cd7d93b, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/63d1d411b03548ff9b69d79afda708e4, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/0e1b8dfe07264766ad283b8e17eb25cc, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/4e9b46dc3b6941879ba25734fb1bbc2a, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/0745d1a501de4885bf8fe6084a730058, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/b17bb20c54714d45945105033f843928, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/5bd4292914e24444b07d971cc6d6af0f, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/4eaffac0042e411e9fb18a4ebd7a50f4, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/791c5f4f0c4443ef8135a8bf0d633b81, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/1d3a1e30ada54cf785b89fb745d5435a, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/9818dd23e06744c2b3553303d72882a4, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/0a5d6b933542435e9bdfd431a48e3b9f, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/9db1b696a60b4ab0a27eb28180b81c09, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/a9c9c7505b0f430590394fca8386bd31, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/ba6a47187d6542188130ed865992a3ee, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/b5a7bb56b6f046e0bafb3bb589033458, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/321752b8ea214a9f96fdf1f6ff4f95a4, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/cb7b7d4d1a124c8eae05b690aab3d1b2, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/c5a9ebf28fe348389cd178d167fddb4b, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/b5c1982a88124ab99cd0fa7b69c1b12a, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/929fae1f3c424403856c609296901bcf, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/db633be2781b4b06993c353c548e49e4, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/ad3750a047d4495eba007673f2259cc4, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/15367e8586e34407b8097f734c1bf8aa, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/0d34644c817e4b1b911c193125ce84b8, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/654017f7ceeb45ab81c634cae286219c] to archive 2024-11-08T00:25:39,630 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T00:25:39,632 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/3ade3e156ded429a8d3306068b0912fa to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/3ade3e156ded429a8d3306068b0912fa 2024-11-08T00:25:39,633 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/9001e2bc424d493399048645ccc55b6c to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/9001e2bc424d493399048645ccc55b6c 2024-11-08T00:25:39,634 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/abded7b318194d89a9e4ff8268fa1d1f to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/abded7b318194d89a9e4ff8268fa1d1f 2024-11-08T00:25:39,635 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/b9a34facbd464d91bebd049fa5c8fc17 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/b9a34facbd464d91bebd049fa5c8fc17 2024-11-08T00:25:39,636 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/947c6958e4ca4dec97be3f4961bdc951 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/947c6958e4ca4dec97be3f4961bdc951 2024-11-08T00:25:39,636 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/8237934a842645a1a175adabf738c223 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/8237934a842645a1a175adabf738c223 2024-11-08T00:25:39,637 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/3d637170ec014ab58f0cc2b63cd7d93b to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/3d637170ec014ab58f0cc2b63cd7d93b 2024-11-08T00:25:39,638 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/63d1d411b03548ff9b69d79afda708e4 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/63d1d411b03548ff9b69d79afda708e4 2024-11-08T00:25:39,638 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/0e1b8dfe07264766ad283b8e17eb25cc to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/0e1b8dfe07264766ad283b8e17eb25cc 2024-11-08T00:25:39,639 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/4e9b46dc3b6941879ba25734fb1bbc2a to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/4e9b46dc3b6941879ba25734fb1bbc2a 2024-11-08T00:25:39,640 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/0745d1a501de4885bf8fe6084a730058 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/0745d1a501de4885bf8fe6084a730058 2024-11-08T00:25:39,641 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/b17bb20c54714d45945105033f843928 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/b17bb20c54714d45945105033f843928 2024-11-08T00:25:39,642 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/5bd4292914e24444b07d971cc6d6af0f to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/5bd4292914e24444b07d971cc6d6af0f 2024-11-08T00:25:39,643 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/4eaffac0042e411e9fb18a4ebd7a50f4 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/4eaffac0042e411e9fb18a4ebd7a50f4 2024-11-08T00:25:39,644 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/791c5f4f0c4443ef8135a8bf0d633b81 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/791c5f4f0c4443ef8135a8bf0d633b81 2024-11-08T00:25:39,645 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/1d3a1e30ada54cf785b89fb745d5435a to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/1d3a1e30ada54cf785b89fb745d5435a 2024-11-08T00:25:39,646 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/9818dd23e06744c2b3553303d72882a4 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/9818dd23e06744c2b3553303d72882a4 2024-11-08T00:25:39,647 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/0a5d6b933542435e9bdfd431a48e3b9f to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/0a5d6b933542435e9bdfd431a48e3b9f 2024-11-08T00:25:39,648 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/9db1b696a60b4ab0a27eb28180b81c09 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/9db1b696a60b4ab0a27eb28180b81c09 2024-11-08T00:25:39,649 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/a9c9c7505b0f430590394fca8386bd31 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/a9c9c7505b0f430590394fca8386bd31 2024-11-08T00:25:39,650 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/ba6a47187d6542188130ed865992a3ee to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/ba6a47187d6542188130ed865992a3ee 2024-11-08T00:25:39,651 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/b5a7bb56b6f046e0bafb3bb589033458 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/b5a7bb56b6f046e0bafb3bb589033458 2024-11-08T00:25:39,652 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/321752b8ea214a9f96fdf1f6ff4f95a4 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/321752b8ea214a9f96fdf1f6ff4f95a4 2024-11-08T00:25:39,653 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/cb7b7d4d1a124c8eae05b690aab3d1b2 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/cb7b7d4d1a124c8eae05b690aab3d1b2 2024-11-08T00:25:39,654 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/c5a9ebf28fe348389cd178d167fddb4b to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/c5a9ebf28fe348389cd178d167fddb4b 2024-11-08T00:25:39,655 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/b5c1982a88124ab99cd0fa7b69c1b12a to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/b5c1982a88124ab99cd0fa7b69c1b12a 2024-11-08T00:25:39,656 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/929fae1f3c424403856c609296901bcf to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/929fae1f3c424403856c609296901bcf 2024-11-08T00:25:39,658 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/db633be2781b4b06993c353c548e49e4 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/db633be2781b4b06993c353c548e49e4 2024-11-08T00:25:39,659 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/ad3750a047d4495eba007673f2259cc4 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/ad3750a047d4495eba007673f2259cc4 2024-11-08T00:25:39,660 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/15367e8586e34407b8097f734c1bf8aa to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/15367e8586e34407b8097f734c1bf8aa 2024-11-08T00:25:39,661 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/0d34644c817e4b1b911c193125ce84b8 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/0d34644c817e4b1b911c193125ce84b8 2024-11-08T00:25:39,662 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/654017f7ceeb45ab81c634cae286219c to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/654017f7ceeb45ab81c634cae286219c 2024-11-08T00:25:39,668 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/d7c75a50452244d3bd4afafd593b01f7, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/03be9bcc09d14e96957aacda5a946263, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/abcb5f77e86c4eb6a1a831c79dbec325, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/ac6303b547894d1583a0bae39b6d0341, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/e35e53b5e42040a4aa695113b7a3a505, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/dee6a9ab03ba485eac3eba2eb836a322, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/4c1e27074a6049098d2dba491f075637, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/f2b004c3f1324e808ab771e269a7004d, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/bf95005ada4245d5988bec7323cc253c, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/e6f298eacc744444bf3c1c7cf23d1ca9, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/c5c0fe94e90b45f3bc9b2b5c7e0d492b, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/f04d8b1d57ec45cd9e071605d501c116, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/2453a3d6405b4208b74fc357e1a04ef2, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/00ca2c40dbff43d4bec5d60944e47bf4, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/c1c8e298011a442a858619408a770181, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/88176bb8ed724ae2a4d554117f1e3d65, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/787d05fcb686498b827264bba6fe9e26, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/58913a3b9c25496ca6cf638d1e08c914, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/e3210d33664c4bb2838ea6bb1c9888f4, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/2935adb519044e94b13b50703dc54387, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/7e51bd399c694121b785c3237222d6d0, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/05ad2918b67b4840937dd29168b0f6f6, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/771fbd0a19264e53a16949e62aa5552d, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/7ed8f60b900144088a51135fbb7ff9e2, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/2faf0ac124ef43579d6b4f3277922cad, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/3bb2e114e9cf4bb2b7137cd736e336bf, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/9cb114ebc8654ac88cf526b88a9571d0, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/e590f4a3267c4d119195f366b6136bd2, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/ad5d4dfadef04907b3ce368e4460ee9c, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/346bf136876544df852b906d17971f76, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/cf373299764b423d871ea331787b7f0d, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/0c7055d4f1654f23b73afe87b683eca6] to archive 2024-11-08T00:25:39,669 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T00:25:39,671 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/d7c75a50452244d3bd4afafd593b01f7 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/d7c75a50452244d3bd4afafd593b01f7 2024-11-08T00:25:39,673 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/03be9bcc09d14e96957aacda5a946263 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/03be9bcc09d14e96957aacda5a946263 2024-11-08T00:25:39,675 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/abcb5f77e86c4eb6a1a831c79dbec325 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/abcb5f77e86c4eb6a1a831c79dbec325 2024-11-08T00:25:39,676 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/ac6303b547894d1583a0bae39b6d0341 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/ac6303b547894d1583a0bae39b6d0341 2024-11-08T00:25:39,677 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/e35e53b5e42040a4aa695113b7a3a505 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/e35e53b5e42040a4aa695113b7a3a505 2024-11-08T00:25:39,678 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/dee6a9ab03ba485eac3eba2eb836a322 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/dee6a9ab03ba485eac3eba2eb836a322 2024-11-08T00:25:39,680 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/4c1e27074a6049098d2dba491f075637 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/4c1e27074a6049098d2dba491f075637 2024-11-08T00:25:39,681 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/f2b004c3f1324e808ab771e269a7004d to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/f2b004c3f1324e808ab771e269a7004d 2024-11-08T00:25:39,682 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/bf95005ada4245d5988bec7323cc253c to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/bf95005ada4245d5988bec7323cc253c 2024-11-08T00:25:39,683 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/e6f298eacc744444bf3c1c7cf23d1ca9 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/e6f298eacc744444bf3c1c7cf23d1ca9 2024-11-08T00:25:39,684 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/c5c0fe94e90b45f3bc9b2b5c7e0d492b to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/c5c0fe94e90b45f3bc9b2b5c7e0d492b 2024-11-08T00:25:39,684 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/f04d8b1d57ec45cd9e071605d501c116 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/f04d8b1d57ec45cd9e071605d501c116 2024-11-08T00:25:39,685 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/2453a3d6405b4208b74fc357e1a04ef2 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/2453a3d6405b4208b74fc357e1a04ef2 2024-11-08T00:25:39,686 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/00ca2c40dbff43d4bec5d60944e47bf4 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/00ca2c40dbff43d4bec5d60944e47bf4 2024-11-08T00:25:39,687 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/c1c8e298011a442a858619408a770181 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/c1c8e298011a442a858619408a770181 2024-11-08T00:25:39,688 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/88176bb8ed724ae2a4d554117f1e3d65 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/88176bb8ed724ae2a4d554117f1e3d65 2024-11-08T00:25:39,689 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/787d05fcb686498b827264bba6fe9e26 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/787d05fcb686498b827264bba6fe9e26 2024-11-08T00:25:39,690 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/58913a3b9c25496ca6cf638d1e08c914 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/58913a3b9c25496ca6cf638d1e08c914 2024-11-08T00:25:39,691 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/e3210d33664c4bb2838ea6bb1c9888f4 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/e3210d33664c4bb2838ea6bb1c9888f4 2024-11-08T00:25:39,692 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/2935adb519044e94b13b50703dc54387 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/2935adb519044e94b13b50703dc54387 2024-11-08T00:25:39,693 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/7e51bd399c694121b785c3237222d6d0 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/7e51bd399c694121b785c3237222d6d0 2024-11-08T00:25:39,694 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/05ad2918b67b4840937dd29168b0f6f6 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/05ad2918b67b4840937dd29168b0f6f6 2024-11-08T00:25:39,696 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/771fbd0a19264e53a16949e62aa5552d to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/771fbd0a19264e53a16949e62aa5552d 2024-11-08T00:25:39,697 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/7ed8f60b900144088a51135fbb7ff9e2 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/7ed8f60b900144088a51135fbb7ff9e2 2024-11-08T00:25:39,698 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/2faf0ac124ef43579d6b4f3277922cad to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/2faf0ac124ef43579d6b4f3277922cad 2024-11-08T00:25:39,699 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/3bb2e114e9cf4bb2b7137cd736e336bf to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/3bb2e114e9cf4bb2b7137cd736e336bf 2024-11-08T00:25:39,700 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/9cb114ebc8654ac88cf526b88a9571d0 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/9cb114ebc8654ac88cf526b88a9571d0 2024-11-08T00:25:39,701 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/e590f4a3267c4d119195f366b6136bd2 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/e590f4a3267c4d119195f366b6136bd2 2024-11-08T00:25:39,703 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/ad5d4dfadef04907b3ce368e4460ee9c to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/ad5d4dfadef04907b3ce368e4460ee9c 2024-11-08T00:25:39,704 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/346bf136876544df852b906d17971f76 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/346bf136876544df852b906d17971f76 2024-11-08T00:25:39,706 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/cf373299764b423d871ea331787b7f0d to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/cf373299764b423d871ea331787b7f0d 2024-11-08T00:25:39,707 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/0c7055d4f1654f23b73afe87b683eca6 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/0c7055d4f1654f23b73afe87b683eca6 2024-11-08T00:25:39,713 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/09198b3429364946bc72ce17f59b79cb, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/e7eb2546db85401da92df02f008a35fd, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/c706c778e9914f1c91ee3d055aa2960d, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/704b0fe277bc4b41bb09c5fdddc8946b, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/4166d9ffae7b4120b463c6dbe3369cb2, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/423de780238b4f13ad857802acf3d785, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/4d8e0cb72dcd45c6afe171658e4db83f, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/2b043bb214394c1bb8830f3657581ae9, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/e75c4c42b8a74539a4934683529e6704, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/3874f50494bd47df85156830d23c0ace, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/1649b56b39954ec8bc22097635f1050c, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/81faea1e3f6c446fb2bf982e92e3f965, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/a570989d8d124566bec065cd6fe4f9d4, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/ed2eb1f4e9c64a63866c4a70b00c52f1, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/06b10b58ddc4474c950e6fb116dbbc86, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/a0257f17ee3241a787dac005f8134dc2, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/18476e3f6aef4ba7b23d38dbd8486667, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/45d6270537ad466499d8e86c6691722e, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/6b571238b4dd40419bf19a14b3a0c863, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/01d8d4151e754493a4cd5dfce3c4a90f, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/86cf1460b3654785b01bc5057207e2c0, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/91484d2347c64dad8fa0a8db38142a23, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/939518c2cffd4893a17d4ed07eb33787, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/578701be7bd54befb29865d660bcf10c, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/b2bf439f05eb427daf4cb9212751508b, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/81c651a7445e4107ae09e5aa8aa7f5b2, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/a2982a612631460c94350015ba1a984a, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/f7fd1ced3afb4605882855ca99eed005, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/ac23323a26ce4b45889f84699f96db84, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/0fcf5e87825b46c1b833e134a1bd23cb, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/8651f233a1334553929c686943db49f0] to archive 2024-11-08T00:25:39,713 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T00:25:39,714 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/09198b3429364946bc72ce17f59b79cb to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/09198b3429364946bc72ce17f59b79cb 2024-11-08T00:25:39,715 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/e7eb2546db85401da92df02f008a35fd to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/e7eb2546db85401da92df02f008a35fd 2024-11-08T00:25:39,716 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/c706c778e9914f1c91ee3d055aa2960d to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/c706c778e9914f1c91ee3d055aa2960d 2024-11-08T00:25:39,717 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/704b0fe277bc4b41bb09c5fdddc8946b to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/704b0fe277bc4b41bb09c5fdddc8946b 2024-11-08T00:25:39,717 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/4166d9ffae7b4120b463c6dbe3369cb2 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/4166d9ffae7b4120b463c6dbe3369cb2 2024-11-08T00:25:39,718 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/423de780238b4f13ad857802acf3d785 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/423de780238b4f13ad857802acf3d785 2024-11-08T00:25:39,718 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/4d8e0cb72dcd45c6afe171658e4db83f to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/4d8e0cb72dcd45c6afe171658e4db83f 2024-11-08T00:25:39,719 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/2b043bb214394c1bb8830f3657581ae9 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/2b043bb214394c1bb8830f3657581ae9 2024-11-08T00:25:39,720 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/e75c4c42b8a74539a4934683529e6704 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/e75c4c42b8a74539a4934683529e6704 2024-11-08T00:25:39,720 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/3874f50494bd47df85156830d23c0ace to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/3874f50494bd47df85156830d23c0ace 2024-11-08T00:25:39,721 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/1649b56b39954ec8bc22097635f1050c to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/1649b56b39954ec8bc22097635f1050c 2024-11-08T00:25:39,722 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/81faea1e3f6c446fb2bf982e92e3f965 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/81faea1e3f6c446fb2bf982e92e3f965 2024-11-08T00:25:39,722 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/a570989d8d124566bec065cd6fe4f9d4 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/a570989d8d124566bec065cd6fe4f9d4 2024-11-08T00:25:39,723 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/ed2eb1f4e9c64a63866c4a70b00c52f1 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/ed2eb1f4e9c64a63866c4a70b00c52f1 2024-11-08T00:25:39,723 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/06b10b58ddc4474c950e6fb116dbbc86 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/06b10b58ddc4474c950e6fb116dbbc86 2024-11-08T00:25:39,724 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/a0257f17ee3241a787dac005f8134dc2 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/a0257f17ee3241a787dac005f8134dc2 2024-11-08T00:25:39,725 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/18476e3f6aef4ba7b23d38dbd8486667 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/18476e3f6aef4ba7b23d38dbd8486667 2024-11-08T00:25:39,726 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/45d6270537ad466499d8e86c6691722e to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/45d6270537ad466499d8e86c6691722e 2024-11-08T00:25:39,726 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/6b571238b4dd40419bf19a14b3a0c863 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/6b571238b4dd40419bf19a14b3a0c863 2024-11-08T00:25:39,727 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/01d8d4151e754493a4cd5dfce3c4a90f to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/01d8d4151e754493a4cd5dfce3c4a90f 2024-11-08T00:25:39,728 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/86cf1460b3654785b01bc5057207e2c0 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/86cf1460b3654785b01bc5057207e2c0 2024-11-08T00:25:39,728 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/91484d2347c64dad8fa0a8db38142a23 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/91484d2347c64dad8fa0a8db38142a23 2024-11-08T00:25:39,729 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/939518c2cffd4893a17d4ed07eb33787 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/939518c2cffd4893a17d4ed07eb33787 2024-11-08T00:25:39,730 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/578701be7bd54befb29865d660bcf10c to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/578701be7bd54befb29865d660bcf10c 2024-11-08T00:25:39,730 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/b2bf439f05eb427daf4cb9212751508b to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/b2bf439f05eb427daf4cb9212751508b 2024-11-08T00:25:39,731 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/81c651a7445e4107ae09e5aa8aa7f5b2 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/81c651a7445e4107ae09e5aa8aa7f5b2 2024-11-08T00:25:39,731 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/a2982a612631460c94350015ba1a984a to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/a2982a612631460c94350015ba1a984a 2024-11-08T00:25:39,732 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/f7fd1ced3afb4605882855ca99eed005 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/f7fd1ced3afb4605882855ca99eed005 2024-11-08T00:25:39,733 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/ac23323a26ce4b45889f84699f96db84 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/ac23323a26ce4b45889f84699f96db84 2024-11-08T00:25:39,733 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/0fcf5e87825b46c1b833e134a1bd23cb to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/0fcf5e87825b46c1b833e134a1bd23cb 2024-11-08T00:25:39,734 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/3302f0f507bd:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/8651f233a1334553929c686943db49f0 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/8651f233a1334553929c686943db49f0 2024-11-08T00:25:41,991 INFO [Thread-1678 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1678. 2024-11-08T00:25:41,991 DEBUG [Thread-1678 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T00:25:41,991 DEBUG [Thread-1678 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:41,991 DEBUG [Thread-1678 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:41,991 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:25:41,992 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-08T00:25:41,992 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2024-11-08T00:25:41,992 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 75 2024-11-08T00:25:41,992 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 77 2024-11-08T00:25:41,992 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 83 2024-11-08T00:25:41,992 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-11-08T00:25:41,992 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-08T00:25:41,992 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-08T00:25:41,992 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 987 2024-11-08T00:25:41,992 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2961 rows 2024-11-08T00:25:41,993 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 984 2024-11-08T00:25:41,993 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2948 rows 2024-11-08T00:25:41,993 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 978 2024-11-08T00:25:41,993 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2921 rows 2024-11-08T00:25:41,993 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 982 2024-11-08T00:25:41,993 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2943 rows 2024-11-08T00:25:41,993 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 975 2024-11-08T00:25:41,993 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2925 rows 2024-11-08T00:25:41,993 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T00:25:41,993 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T00:25:41,994 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:41,994 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:41,994 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:25:41,996 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-08T00:25:41,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-08T00:25:42,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-08T00:25:42,001 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731025542001"}]},"ts":"1731025542001"} 2024-11-08T00:25:42,003 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-08T00:25:42,003 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-08T00:25:42,004 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-08T00:25:42,006 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0d1e33fe1a366293d5e32ce198366168, UNASSIGN}] 2024-11-08T00:25:42,007 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0d1e33fe1a366293d5e32ce198366168, UNASSIGN 2024-11-08T00:25:42,008 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=0d1e33fe1a366293d5e32ce198366168, regionState=CLOSING, regionLocation=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:42,009 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0d1e33fe1a366293d5e32ce198366168, UNASSIGN because future has completed 2024-11-08T00:25:42,009 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-08T00:25:42,010 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856}] 2024-11-08T00:25:42,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-08T00:25:42,162 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] handler.UnassignRegionHandler(122): Close 0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:42,162 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-08T00:25:42,163 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1722): Closing 0d1e33fe1a366293d5e32ce198366168, disabling compactions & flushes 2024-11-08T00:25:42,163 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:42,163 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:42,163 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. after waiting 0 ms 2024-11-08T00:25:42,163 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:42,163 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(2902): Flushing 0d1e33fe1a366293d5e32ce198366168 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-08T00:25:42,172 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411084e36f3ae75094c7ea76164e723201705_0d1e33fe1a366293d5e32ce198366168 is 50, key is test_row_0/A:col10/1731025532778/Put/seqid=0 2024-11-08T00:25:42,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742330_1506 (size=12454) 2024-11-08T00:25:42,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-08T00:25:42,577 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:42,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-08T00:25:42,630 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411084e36f3ae75094c7ea76164e723201705_0d1e33fe1a366293d5e32ce198366168 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411084e36f3ae75094c7ea76164e723201705_0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:42,632 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/c9facad18e5348fe8d83afa12443a5af, store: [table=TestAcidGuarantees family=A region=0d1e33fe1a366293d5e32ce198366168] 2024-11-08T00:25:42,632 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/c9facad18e5348fe8d83afa12443a5af is 175, key is test_row_0/A:col10/1731025532778/Put/seqid=0 2024-11-08T00:25:42,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742331_1507 (size=31255) 2024-11-08T00:25:43,039 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=489, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/c9facad18e5348fe8d83afa12443a5af 2024-11-08T00:25:43,049 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/eb6048b1847841df9cfa78bf54fbf634 is 50, key is test_row_0/B:col10/1731025532778/Put/seqid=0 2024-11-08T00:25:43,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742332_1508 (size=12301) 2024-11-08T00:25:43,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-08T00:25:43,454 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=489 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/eb6048b1847841df9cfa78bf54fbf634 2024-11-08T00:25:43,465 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/d584628d12f649e0ab51341636818d5e is 50, key is test_row_0/C:col10/1731025532778/Put/seqid=0 2024-11-08T00:25:43,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742333_1509 (size=12301) 2024-11-08T00:25:43,871 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=489 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/d584628d12f649e0ab51341636818d5e 2024-11-08T00:25:43,877 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/A/c9facad18e5348fe8d83afa12443a5af as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/c9facad18e5348fe8d83afa12443a5af 2024-11-08T00:25:43,880 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/c9facad18e5348fe8d83afa12443a5af, entries=150, sequenceid=489, filesize=30.5 K 2024-11-08T00:25:43,881 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/B/eb6048b1847841df9cfa78bf54fbf634 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/eb6048b1847841df9cfa78bf54fbf634 2024-11-08T00:25:43,884 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/eb6048b1847841df9cfa78bf54fbf634, entries=150, sequenceid=489, filesize=12.0 K 2024-11-08T00:25:43,885 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/.tmp/C/d584628d12f649e0ab51341636818d5e as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/d584628d12f649e0ab51341636818d5e 2024-11-08T00:25:43,888 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/d584628d12f649e0ab51341636818d5e, entries=150, sequenceid=489, filesize=12.0 K 2024-11-08T00:25:43,889 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 0d1e33fe1a366293d5e32ce198366168 in 1726ms, sequenceid=489, compaction requested=true 2024-11-08T00:25:43,892 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/recovered.edits/492.seqid, newMaxSeqId=492, maxSeqId=4 2024-11-08T00:25:43,893 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. 2024-11-08T00:25:43,893 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1676): Region close journal for 0d1e33fe1a366293d5e32ce198366168: Waiting for close lock at 1731025542162Running coprocessor pre-close hooks at 1731025542162Disabling compacts and flushes for region at 1731025542162Disabling writes for close at 1731025542163 (+1 ms)Obtaining lock to block concurrent updates at 1731025542163Preparing flush snapshotting stores in 0d1e33fe1a366293d5e32ce198366168 at 1731025542163Finished memstore snapshotting TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731025542163Flushing stores of TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168. at 1731025542164 (+1 ms)Flushing 0d1e33fe1a366293d5e32ce198366168/A: creating writer at 1731025542164Flushing mob file 0d1e33fe1a366293d5e32ce198366168/A: appending metadata at 1731025542172 (+8 ms)Flushing mob file 0d1e33fe1a366293d5e32ce198366168/A: closing flushed file at 1731025542172Flushing 0d1e33fe1a366293d5e32ce198366168/A: appending metadata at 1731025542632 (+460 ms)Flushing 0d1e33fe1a366293d5e32ce198366168/A: closing flushed file at 1731025542632Flushing 0d1e33fe1a366293d5e32ce198366168/B: creating writer at 1731025543045 (+413 ms)Flushing 0d1e33fe1a366293d5e32ce198366168/B: appending metadata at 1731025543049 (+4 ms)Flushing 0d1e33fe1a366293d5e32ce198366168/B: closing flushed file at 1731025543049Flushing 0d1e33fe1a366293d5e32ce198366168/C: creating writer at 1731025543461 (+412 ms)Flushing 0d1e33fe1a366293d5e32ce198366168/C: appending metadata at 1731025543465 (+4 ms)Flushing 0d1e33fe1a366293d5e32ce198366168/C: closing flushed file at 1731025543465Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5ac431b8: reopening flushed file at 1731025543876 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2e382b2c: reopening flushed file at 1731025543880 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1fa385c2: reopening flushed file at 1731025543884 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 0d1e33fe1a366293d5e32ce198366168 in 1726ms, sequenceid=489, compaction requested=true at 1731025543889 (+5 ms)Writing region close event to WAL at 1731025543890 (+1 ms)Running coprocessor post-close hooks at 1731025543893 (+3 ms)Closed at 1731025543893 2024-11-08T00:25:43,895 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] handler.UnassignRegionHandler(157): Closed 0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:43,895 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=0d1e33fe1a366293d5e32ce198366168, regionState=CLOSED 2024-11-08T00:25:43,896 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=118, ppid=117, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 because future has completed 2024-11-08T00:25:43,898 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-08T00:25:43,898 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; CloseRegionProcedure 0d1e33fe1a366293d5e32ce198366168, server=3302f0f507bd,40479,1731025415856 in 1.8870 sec 2024-11-08T00:25:43,899 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=117, resume processing ppid=116 2024-11-08T00:25:43,900 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, ppid=116, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0d1e33fe1a366293d5e32ce198366168, UNASSIGN in 1.8920 sec 2024-11-08T00:25:43,901 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-08T00:25:43,901 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8960 sec 2024-11-08T00:25:43,902 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731025543902"}]},"ts":"1731025543902"} 2024-11-08T00:25:43,903 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-08T00:25:43,903 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-08T00:25:43,905 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9080 sec 2024-11-08T00:25:44,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-08T00:25:44,148 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:25:44,148 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-08T00:25:44,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T00:25:44,150 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=119, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T00:25:44,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-08T00:25:44,150 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=119, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T00:25:44,153 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:44,155 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A, FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B, FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C, FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/recovered.edits] 2024-11-08T00:25:44,158 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/56e2138691914096892edd06bd8e4c26 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/56e2138691914096892edd06bd8e4c26 2024-11-08T00:25:44,160 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/c9facad18e5348fe8d83afa12443a5af to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/c9facad18e5348fe8d83afa12443a5af 2024-11-08T00:25:44,161 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/e9472ac2f11c473e946b8689999abb7f to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/A/e9472ac2f11c473e946b8689999abb7f 2024-11-08T00:25:44,164 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/11de1e334de24ffeb7d1b33360cddd0b to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/11de1e334de24ffeb7d1b33360cddd0b 2024-11-08T00:25:44,165 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/4ba5c337c2c0456ba935272998884427 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/4ba5c337c2c0456ba935272998884427 2024-11-08T00:25:44,166 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/eb6048b1847841df9cfa78bf54fbf634 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/B/eb6048b1847841df9cfa78bf54fbf634 2024-11-08T00:25:44,168 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/42869447eade4dd9899fc8f74f324e3e to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/42869447eade4dd9899fc8f74f324e3e 2024-11-08T00:25:44,169 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/7c131c56123c4430bc4b587eabcd34e6 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/7c131c56123c4430bc4b587eabcd34e6 2024-11-08T00:25:44,169 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/d584628d12f649e0ab51341636818d5e to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/C/d584628d12f649e0ab51341636818d5e 2024-11-08T00:25:44,171 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/recovered.edits/492.seqid to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168/recovered.edits/492.seqid 2024-11-08T00:25:44,171 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:44,171 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-08T00:25:44,172 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-08T00:25:44,172 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-08T00:25:44,175 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110801fec44bc1e34f56b5e367de30a7ade2_0d1e33fe1a366293d5e32ce198366168 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110801fec44bc1e34f56b5e367de30a7ade2_0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:44,176 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110805d399f545424b5b84f12025c176d4b7_0d1e33fe1a366293d5e32ce198366168 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110805d399f545424b5b84f12025c176d4b7_0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:44,177 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110806a62b7cf28346ad8b4578b267ed2d52_0d1e33fe1a366293d5e32ce198366168 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110806a62b7cf28346ad8b4578b267ed2d52_0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:44,178 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411081046c4dd056d4c4f8916ebed4f5d1845_0d1e33fe1a366293d5e32ce198366168 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411081046c4dd056d4c4f8916ebed4f5d1845_0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:44,179 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108122fb3faefdc47c0b68ba5a072befa78_0d1e33fe1a366293d5e32ce198366168 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108122fb3faefdc47c0b68ba5a072befa78_0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:44,180 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110814b82a4967ea4eafa828376776e17f48_0d1e33fe1a366293d5e32ce198366168 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110814b82a4967ea4eafa828376776e17f48_0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:44,180 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411083c5bf1e608de49fcace955829cbf05cb_0d1e33fe1a366293d5e32ce198366168 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411083c5bf1e608de49fcace955829cbf05cb_0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:44,181 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108428fa3b3477c4118ba783d58e33f3f12_0d1e33fe1a366293d5e32ce198366168 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108428fa3b3477c4118ba783d58e33f3f12_0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:44,182 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108429dfede8b02420aadc7a57b5053bdd9_0d1e33fe1a366293d5e32ce198366168 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108429dfede8b02420aadc7a57b5053bdd9_0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:44,183 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411084e36f3ae75094c7ea76164e723201705_0d1e33fe1a366293d5e32ce198366168 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411084e36f3ae75094c7ea76164e723201705_0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:44,184 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108548329b68be8417597893de2609d3d03_0d1e33fe1a366293d5e32ce198366168 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108548329b68be8417597893de2609d3d03_0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:44,186 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411085dc0f4aa26594acabf971b1f4d4d8f37_0d1e33fe1a366293d5e32ce198366168 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411085dc0f4aa26594acabf971b1f4d4d8f37_0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:44,187 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411087359cd17d9394989b9dcfadf6ab8ebc8_0d1e33fe1a366293d5e32ce198366168 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411087359cd17d9394989b9dcfadf6ab8ebc8_0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:44,188 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110878fa4cb07b1d476bb0098ca3fc60b0a3_0d1e33fe1a366293d5e32ce198366168 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110878fa4cb07b1d476bb0098ca3fc60b0a3_0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:44,189 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411087d4d8fce9e794c68b174f844042b57e6_0d1e33fe1a366293d5e32ce198366168 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411087d4d8fce9e794c68b174f844042b57e6_0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:44,190 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411088e35ecd5cc60442c99b4cd33c913d583_0d1e33fe1a366293d5e32ce198366168 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411088e35ecd5cc60442c99b4cd33c913d583_0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:44,191 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411088f1a9c69bf8a43f9bc58678632e8e3ca_0d1e33fe1a366293d5e32ce198366168 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411088f1a9c69bf8a43f9bc58678632e8e3ca_0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:44,192 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108938d75b3f3184bba9a46b9d175be8042_0d1e33fe1a366293d5e32ce198366168 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108938d75b3f3184bba9a46b9d175be8042_0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:44,193 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108a5b30f7f4ae54c398040e126824e10a3_0d1e33fe1a366293d5e32ce198366168 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108a5b30f7f4ae54c398040e126824e10a3_0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:44,194 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108af4562cf07294a298715048c508ddb0d_0d1e33fe1a366293d5e32ce198366168 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108af4562cf07294a298715048c508ddb0d_0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:44,194 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108b257c660cf19402c84b89e70731046dd_0d1e33fe1a366293d5e32ce198366168 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108b257c660cf19402c84b89e70731046dd_0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:44,195 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108bd08789c74ad476f99eee08a8a92b5a0_0d1e33fe1a366293d5e32ce198366168 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108bd08789c74ad476f99eee08a8a92b5a0_0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:44,196 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108c3b501476f5a43e7adb66446fb1fb5db_0d1e33fe1a366293d5e32ce198366168 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108c3b501476f5a43e7adb66446fb1fb5db_0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:44,197 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108c73ca7db72d942d2a33f6556dc25012e_0d1e33fe1a366293d5e32ce198366168 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108c73ca7db72d942d2a33f6556dc25012e_0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:44,198 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108cba703e603164a0f90a31f9e5a1778ec_0d1e33fe1a366293d5e32ce198366168 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108cba703e603164a0f90a31f9e5a1778ec_0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:44,198 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108d2934e4655ff4faca1a5180bf00a8647_0d1e33fe1a366293d5e32ce198366168 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108d2934e4655ff4faca1a5180bf00a8647_0d1e33fe1a366293d5e32ce198366168 2024-11-08T00:25:44,199 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-08T00:25:44,201 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=119, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T00:25:44,203 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-08T00:25:44,205 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-08T00:25:44,206 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=119, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T00:25:44,206 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-08T00:25:44,207 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731025544206"}]},"ts":"9223372036854775807"} 2024-11-08T00:25:44,209 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-08T00:25:44,209 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 0d1e33fe1a366293d5e32ce198366168, NAME => 'TestAcidGuarantees,,1731025508986.0d1e33fe1a366293d5e32ce198366168.', STARTKEY => '', ENDKEY => ''}] 2024-11-08T00:25:44,209 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-08T00:25:44,209 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731025544209"}]},"ts":"9223372036854775807"} 2024-11-08T00:25:44,211 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-08T00:25:44,212 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=119, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T00:25:44,213 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 64 msec 2024-11-08T00:25:44,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-08T00:25:44,258 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-08T00:25:44,258 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:25:44,272 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=238 (was 239), OpenFileDescriptor=445 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=666 (was 785), ProcessCount=11 (was 11), AvailableMemoryMB=6426 (was 4397) - AvailableMemoryMB LEAK? - 2024-11-08T00:25:44,280 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=238, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=666, ProcessCount=12, AvailableMemoryMB=6425 2024-11-08T00:25:44,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-08T00:25:44,281 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T00:25:44,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=120, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-08T00:25:44,283 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T00:25:44,283 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:44,283 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 120 2024-11-08T00:25:44,284 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T00:25:44,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-08T00:25:44,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742334_1510 (size=959) 2024-11-08T00:25:44,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-08T00:25:44,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-08T00:25:44,691 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f 2024-11-08T00:25:44,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742335_1511 (size=53) 2024-11-08T00:25:44,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-08T00:25:45,053 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-08T00:25:45,100 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:25:45,100 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 04acb16af9bbc0bb9b137adb2146a06b, disabling compactions & flushes 2024-11-08T00:25:45,100 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:45,100 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:45,100 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. after waiting 0 ms 2024-11-08T00:25:45,100 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:45,100 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:45,100 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 04acb16af9bbc0bb9b137adb2146a06b: Waiting for close lock at 1731025545100Disabling compacts and flushes for region at 1731025545100Disabling writes for close at 1731025545100Writing region close event to WAL at 1731025545100Closed at 1731025545100 2024-11-08T00:25:45,102 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T00:25:45,102 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731025545102"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731025545102"}]},"ts":"1731025545102"} 2024-11-08T00:25:45,104 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-08T00:25:45,105 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T00:25:45,106 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731025545106"}]},"ts":"1731025545106"} 2024-11-08T00:25:45,107 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-08T00:25:45,108 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=04acb16af9bbc0bb9b137adb2146a06b, ASSIGN}] 2024-11-08T00:25:45,109 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=04acb16af9bbc0bb9b137adb2146a06b, ASSIGN 2024-11-08T00:25:45,110 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=04acb16af9bbc0bb9b137adb2146a06b, ASSIGN; state=OFFLINE, location=3302f0f507bd,40479,1731025415856; forceNewPlan=false, retain=false 2024-11-08T00:25:45,261 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=04acb16af9bbc0bb9b137adb2146a06b, regionState=OPENING, regionLocation=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:45,263 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=04acb16af9bbc0bb9b137adb2146a06b, ASSIGN because future has completed 2024-11-08T00:25:45,263 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; OpenRegionProcedure 04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856}] 2024-11-08T00:25:45,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-08T00:25:45,421 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:45,421 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(7752): Opening region: {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} 2024-11-08T00:25:45,423 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 04acb16af9bbc0bb9b137adb2146a06b 2024-11-08T00:25:45,423 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:25:45,423 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(7794): checking encryption for 04acb16af9bbc0bb9b137adb2146a06b 2024-11-08T00:25:45,423 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(7797): checking classloading for 04acb16af9bbc0bb9b137adb2146a06b 2024-11-08T00:25:45,426 INFO [StoreOpener-04acb16af9bbc0bb9b137adb2146a06b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 04acb16af9bbc0bb9b137adb2146a06b 2024-11-08T00:25:45,428 INFO [StoreOpener-04acb16af9bbc0bb9b137adb2146a06b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 04acb16af9bbc0bb9b137adb2146a06b columnFamilyName A 2024-11-08T00:25:45,428 DEBUG [StoreOpener-04acb16af9bbc0bb9b137adb2146a06b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:45,429 INFO [StoreOpener-04acb16af9bbc0bb9b137adb2146a06b-1 {}] regionserver.HStore(327): Store=04acb16af9bbc0bb9b137adb2146a06b/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:25:45,429 INFO [StoreOpener-04acb16af9bbc0bb9b137adb2146a06b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 04acb16af9bbc0bb9b137adb2146a06b 2024-11-08T00:25:45,430 INFO [StoreOpener-04acb16af9bbc0bb9b137adb2146a06b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 04acb16af9bbc0bb9b137adb2146a06b columnFamilyName B 2024-11-08T00:25:45,430 DEBUG [StoreOpener-04acb16af9bbc0bb9b137adb2146a06b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:45,431 INFO [StoreOpener-04acb16af9bbc0bb9b137adb2146a06b-1 {}] regionserver.HStore(327): Store=04acb16af9bbc0bb9b137adb2146a06b/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:25:45,431 INFO [StoreOpener-04acb16af9bbc0bb9b137adb2146a06b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 04acb16af9bbc0bb9b137adb2146a06b 2024-11-08T00:25:45,432 INFO [StoreOpener-04acb16af9bbc0bb9b137adb2146a06b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 04acb16af9bbc0bb9b137adb2146a06b columnFamilyName C 2024-11-08T00:25:45,432 DEBUG [StoreOpener-04acb16af9bbc0bb9b137adb2146a06b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:25:45,432 INFO [StoreOpener-04acb16af9bbc0bb9b137adb2146a06b-1 {}] regionserver.HStore(327): Store=04acb16af9bbc0bb9b137adb2146a06b/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:25:45,432 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1038): replaying wal for 04acb16af9bbc0bb9b137adb2146a06b 2024-11-08T00:25:45,433 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b 2024-11-08T00:25:45,433 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b 2024-11-08T00:25:45,434 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1048): stopping wal replay for 04acb16af9bbc0bb9b137adb2146a06b 2024-11-08T00:25:45,434 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1060): Cleaning up temporary data for 04acb16af9bbc0bb9b137adb2146a06b 2024-11-08T00:25:45,435 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-08T00:25:45,436 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1093): writing seq id for 04acb16af9bbc0bb9b137adb2146a06b 2024-11-08T00:25:45,437 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T00:25:45,438 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1114): Opened 04acb16af9bbc0bb9b137adb2146a06b; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72198863, jitterRate=0.07584689557552338}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-08T00:25:45,438 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 04acb16af9bbc0bb9b137adb2146a06b 2024-11-08T00:25:45,438 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1006): Region open journal for 04acb16af9bbc0bb9b137adb2146a06b: Running coprocessor pre-open hook at 1731025545423Writing region info on filesystem at 1731025545423Initializing all the Stores at 1731025545425 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731025545425Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731025545426 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731025545426Cleaning up temporary data from old regions at 1731025545434 (+8 ms)Running coprocessor post-open hooks at 1731025545438 (+4 ms)Region opened successfully at 1731025545438 2024-11-08T00:25:45,439 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., pid=122, masterSystemTime=1731025545416 2024-11-08T00:25:45,441 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:45,441 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:45,441 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=04acb16af9bbc0bb9b137adb2146a06b, regionState=OPEN, openSeqNum=2, regionLocation=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:45,442 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=122, ppid=121, state=RUNNABLE, hasLock=false; OpenRegionProcedure 04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 because future has completed 2024-11-08T00:25:45,445 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-08T00:25:45,445 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; OpenRegionProcedure 04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 in 180 msec 2024-11-08T00:25:45,447 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=121, resume processing ppid=120 2024-11-08T00:25:45,447 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, ppid=120, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=04acb16af9bbc0bb9b137adb2146a06b, ASSIGN in 337 msec 2024-11-08T00:25:45,447 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T00:25:45,448 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731025545447"}]},"ts":"1731025545447"} 2024-11-08T00:25:45,449 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-08T00:25:45,450 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T00:25:45,452 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1690 sec 2024-11-08T00:25:46,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-08T00:25:46,428 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:25:46,431 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77001fc0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:46,431 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:25:46,431 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:25:46,433 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:25:46,433 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:25:46,434 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:25:46,434 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5af4574c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:46,434 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:25:46,434 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:25:46,435 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:46,436 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49834, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:25:46,437 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34ec29dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:46,438 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:25:46,439 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:25:46,439 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:25:46,439 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50186, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:25:46,440 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-08T00:25:46,441 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 3302f0f507bd,36785,1731025414356 2024-11-08T00:25:46,441 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@14e38fda 2024-11-08T00:25:46,441 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-08T00:25:46,441 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49848, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-08T00:25:46,442 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@414548ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:46,442 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:25:46,443 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:25:46,443 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:25:46,443 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:25:46,443 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:25:46,443 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1276cc24, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:46,443 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:25:46,443 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:25:46,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:46,444 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49862, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:25:46,444 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@490a3bd2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:46,445 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e6f359b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:46,445 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:25:46,445 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:25:46,446 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:25:46,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:25:46,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:25:46,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d35c5e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:46,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:25:46,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:25:46,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:46,447 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49880, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:25:46,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60497270, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:46,447 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@237707e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:46,447 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:25:46,448 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:25:46,448 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:25:46,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:25:46,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:25:46,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59fc5648, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:46,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:25:46,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:25:46,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:46,449 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49898, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:25:46,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@304cc45c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:46,450 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2375345f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:46,450 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:25:46,450 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:25:46,450 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:25:46,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:25:46,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:25:46,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d42aca7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:46,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:25:46,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:25:46,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:46,451 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49906, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:25:46,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@575b89dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:46,453 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@154e075c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:46,453 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:25:46,453 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:25:46,453 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:25:46,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:25:46,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:25:46,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20d605f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:46,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:25:46,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:25:46,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:46,455 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49912, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:25:46,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31007b96, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:46,456 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a899d0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:46,456 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:25:46,456 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:25:46,456 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:25:46,456 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:25:46,457 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:25:46,457 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e49afa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:46,457 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:25:46,457 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:25:46,457 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:46,458 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49918, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:25:46,458 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4de8130f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:46,459 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a736d8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:46,459 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:25:46,459 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:25:46,459 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:25:46,460 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:25:46,460 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:25:46,460 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14e84b3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:46,460 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:25:46,460 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:25:46,460 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:46,461 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49934, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:25:46,461 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24e89d08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:46,462 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@609bc762, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:46,462 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:25:46,462 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:25:46,462 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:25:46,463 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:25:46,463 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:25:46,463 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dfb8d5a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:46,463 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:25:46,463 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:25:46,463 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:46,464 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49944, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:25:46,465 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@203e3cf3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:46,466 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71dab0db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:46,467 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:25:46,467 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:25:46,468 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:25:46,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:25:46,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:25:46,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38fe00bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:46,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:25:46,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:25:46,469 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:46,469 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49972, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:25:46,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@693035f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:46,470 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d9eeb91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:46,470 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:25:46,471 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:25:46,471 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:25:46,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:25:46,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:25:46,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50393032, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:46,472 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:25:46,472 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:25:46,472 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:25:46,473 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49988, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:25:46,473 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@451a766b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:25:46,474 DEBUG [Thread-2271 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:25:46,474 DEBUG [Thread-2282 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:25:46,474 DEBUG [Thread-2280 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:25:46,474 DEBUG [Thread-2278 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:25:46,474 DEBUG [Thread-2265 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:25:46,474 DEBUG [Thread-2267 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:25:46,474 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-08T00:25:46,475 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:25:46,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:25:46,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:25:46,475 DEBUG [Thread-2278 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:25:46,475 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:25:46,475 DEBUG [Thread-2278 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:25:46,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:25:46,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:25:46,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:25:46,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:25:46,475 DEBUG [Thread-2273 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:25:46,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:25:46,476 DEBUG [Thread-2284 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:25:46,476 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:25:46,476 DEBUG [Thread-2269 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:25:46,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-08T00:25:46,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:25:46,476 DEBUG [Thread-2276 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:25:46,476 DEBUG [Thread-2273 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:25:46,477 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:25:46,477 DEBUG [Thread-2273 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:25:46,477 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50208, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:25:46,477 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50204, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:25:46,477 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50194, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:25:46,477 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50236, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:25:46,477 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50238, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:25:46,477 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50222, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:25:46,477 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:25:46,477 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:25:46,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-08T00:25:46,478 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:25:46,478 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:25:46,478 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50248, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:25:46,478 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T00:25:46,478 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50264, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:25:46,478 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50274, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:25:46,479 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T00:25:46,479 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T00:25:46,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2] 2024-11-08T00:25:46,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2] 2024-11-08T00:25:46,480 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50280, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:25:46,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2] 2024-11-08T00:25:46,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2] 2024-11-08T00:25:46,481 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2] 2024-11-08T00:25:46,481 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2] 2024-11-08T00:25:46,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2] 2024-11-08T00:25:46,482 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2] 2024-11-08T00:25:46,482 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2] 2024-11-08T00:25:46,482 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2] 2024-11-08T00:25:46,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 04acb16af9bbc0bb9b137adb2146a06b 2024-11-08T00:25:46,489 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:25:46,489 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 04acb16af9bbc0bb9b137adb2146a06b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T00:25:46,512 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/e407d50c85d64bfc847fa9e4a48292a1 is 50, key is test_row_0/A:col10/1731025546487/Put/seqid=0 2024-11-08T00:25:46,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742336_1512 (size=12001) 2024-11-08T00:25:46,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:46,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50248 deadline: 1731025606552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:46,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:46,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50280 deadline: 1731025606552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:46,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:46,566 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:46,566 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:46,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50222 deadline: 1731025606553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:46,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:46,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:46,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1731025606553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:46,567 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:46,567 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:46,567 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:46,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:46,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50194 deadline: 1731025606554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:46,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:46,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:46,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:46,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:46,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:46,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:46,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:46,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:46,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:46,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-08T00:25:46,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-08T00:25:46,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:46,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. as already flushing 2024-11-08T00:25:46,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:46,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:46,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:46,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:46,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:46,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50248 deadline: 1731025606677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:46,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:46,680 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:46,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50280 deadline: 1731025606677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:46,680 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:46,680 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:46,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:46,680 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:46,680 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:46,680 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:46,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1731025606677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:46,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:46,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:46,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:46,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:46,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50194 deadline: 1731025606677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:46,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:46,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:46,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:46,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50222 deadline: 1731025606677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:46,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:46,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:46,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:46,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:46,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-08T00:25:46,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:46,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. as already flushing 2024-11-08T00:25:46,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:46,787 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:46,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:46,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:46,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-08T00:25:46,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:46,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50248 deadline: 1731025606887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:46,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:46,892 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:46,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50280 deadline: 1731025606887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:46,892 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:46,892 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:46,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:46,893 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:46,893 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:46,893 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:46,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1731025606887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:46,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:46,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:46,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:46,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:46,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50194 deadline: 1731025606887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:46,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:46,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:46,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50222 deadline: 1731025606887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:46,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:46,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:46,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:46,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:46,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:46,919 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/e407d50c85d64bfc847fa9e4a48292a1 2024-11-08T00:25:46,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/c30363cdfa1d4f61a2486afc9809abb6 is 50, key is test_row_0/B:col10/1731025546487/Put/seqid=0 2024-11-08T00:25:46,939 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-08T00:25:46,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:46,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. as already flushing 2024-11-08T00:25:46,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:46,939 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:46,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:46,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:46,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742337_1513 (size=12001) 2024-11-08T00:25:47,092 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-08T00:25:47,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:47,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. as already flushing 2024-11-08T00:25:47,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:47,092 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:47,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:47,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:47,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-08T00:25:47,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:47,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50248 deadline: 1731025607197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:47,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:47,201 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:47,201 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:47,201 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:47,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1731025607197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:47,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:47,201 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:47,201 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:47,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50280 deadline: 1731025607197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:47,201 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:47,201 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:47,201 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:47,201 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:47,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:47,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50194 deadline: 1731025607197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:47,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:47,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:47,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:47,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:47,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50222 deadline: 1731025607197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:47,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:47,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:47,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:47,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-08T00:25:47,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:47,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. as already flushing 2024-11-08T00:25:47,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:47,245 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:47,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:47,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:47,344 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/c30363cdfa1d4f61a2486afc9809abb6 2024-11-08T00:25:47,359 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/66c27b728d084ebda6ad9c8577f694f7 is 50, key is test_row_0/C:col10/1731025546487/Put/seqid=0 2024-11-08T00:25:47,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742338_1514 (size=12001) 2024-11-08T00:25:47,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-08T00:25:47,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:47,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. as already flushing 2024-11-08T00:25:47,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:47,397 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:47,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:47,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:47,549 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-08T00:25:47,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:47,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. as already flushing 2024-11-08T00:25:47,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:47,550 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:47,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:47,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:47,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-08T00:25:47,702 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-08T00:25:47,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:47,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. as already flushing 2024-11-08T00:25:47,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:47,702 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:47,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:47,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:47,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:47,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50248 deadline: 1731025607707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:47,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:47,709 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:47,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1731025607707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:47,709 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:47,709 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:47,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:47,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:47,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:47,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50280 deadline: 1731025607707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:47,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:47,709 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:47,709 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:47,709 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:47,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:47,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50194 deadline: 1731025607708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:47,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:47,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:47,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:47,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:47,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50222 deadline: 1731025607717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:47,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:47,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:47,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:47,763 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/66c27b728d084ebda6ad9c8577f694f7 2024-11-08T00:25:47,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/e407d50c85d64bfc847fa9e4a48292a1 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/e407d50c85d64bfc847fa9e4a48292a1 2024-11-08T00:25:47,768 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/e407d50c85d64bfc847fa9e4a48292a1, entries=150, sequenceid=12, filesize=11.7 K 2024-11-08T00:25:47,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/c30363cdfa1d4f61a2486afc9809abb6 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/c30363cdfa1d4f61a2486afc9809abb6 2024-11-08T00:25:47,771 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/c30363cdfa1d4f61a2486afc9809abb6, entries=150, sequenceid=12, filesize=11.7 K 2024-11-08T00:25:47,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/66c27b728d084ebda6ad9c8577f694f7 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/66c27b728d084ebda6ad9c8577f694f7 2024-11-08T00:25:47,774 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/66c27b728d084ebda6ad9c8577f694f7, entries=150, sequenceid=12, filesize=11.7 K 2024-11-08T00:25:47,775 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 04acb16af9bbc0bb9b137adb2146a06b in 1286ms, sequenceid=12, compaction requested=false 2024-11-08T00:25:47,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 04acb16af9bbc0bb9b137adb2146a06b: 2024-11-08T00:25:47,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-08T00:25:47,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:47,855 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 04acb16af9bbc0bb9b137adb2146a06b 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-08T00:25:47,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/510ec65edb8848b0908a03c84a718e32 is 50, key is test_row_0/A:col10/1731025546552/Put/seqid=0 2024-11-08T00:25:47,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742339_1515 (size=12001) 2024-11-08T00:25:48,269 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/510ec65edb8848b0908a03c84a718e32 2024-11-08T00:25:48,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/72b4450e2a2148b3a9f6e4f3c8750ce4 is 50, key is test_row_0/B:col10/1731025546552/Put/seqid=0 2024-11-08T00:25:48,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742340_1516 (size=12001) 2024-11-08T00:25:48,289 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/72b4450e2a2148b3a9f6e4f3c8750ce4 2024-11-08T00:25:48,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/d8f7538b6cba497f9b87a374193e04ec is 50, key is test_row_0/C:col10/1731025546552/Put/seqid=0 2024-11-08T00:25:48,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742341_1517 (size=12001) 2024-11-08T00:25:48,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-08T00:25:48,707 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/d8f7538b6cba497f9b87a374193e04ec 2024-11-08T00:25:48,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/510ec65edb8848b0908a03c84a718e32 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/510ec65edb8848b0908a03c84a718e32 2024-11-08T00:25:48,713 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/510ec65edb8848b0908a03c84a718e32, entries=150, sequenceid=37, filesize=11.7 K 2024-11-08T00:25:48,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/72b4450e2a2148b3a9f6e4f3c8750ce4 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/72b4450e2a2148b3a9f6e4f3c8750ce4 2024-11-08T00:25:48,716 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/72b4450e2a2148b3a9f6e4f3c8750ce4, entries=150, sequenceid=37, filesize=11.7 K 2024-11-08T00:25:48,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/d8f7538b6cba497f9b87a374193e04ec as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/d8f7538b6cba497f9b87a374193e04ec 2024-11-08T00:25:48,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. as already flushing 2024-11-08T00:25:48,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 04acb16af9bbc0bb9b137adb2146a06b 2024-11-08T00:25:48,721 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/d8f7538b6cba497f9b87a374193e04ec, entries=150, sequenceid=37, filesize=11.7 K 2024-11-08T00:25:48,721 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=13.42 KB/13740 for 04acb16af9bbc0bb9b137adb2146a06b in 866ms, sequenceid=37, compaction requested=false 2024-11-08T00:25:48,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 04acb16af9bbc0bb9b137adb2146a06b: 2024-11-08T00:25:48,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:48,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-08T00:25:48,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-08T00:25:48,725 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-08T00:25:48,725 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2440 sec 2024-11-08T00:25:48,726 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 2.2510 sec 2024-11-08T00:25:48,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 04acb16af9bbc0bb9b137adb2146a06b 2024-11-08T00:25:48,730 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:25:48,730 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 04acb16af9bbc0bb9b137adb2146a06b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T00:25:48,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/9eae889bf07a4633a8ab2fb9c695c310 is 50, key is test_row_0/A:col10/1731025548729/Put/seqid=0 2024-11-08T00:25:48,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742342_1518 (size=19021) 2024-11-08T00:25:48,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:48,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50280 deadline: 1731025608771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:48,777 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:48,777 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:48,777 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:48,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:48,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50248 deadline: 1731025608773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:48,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:48,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:48,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:48,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:48,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50194 deadline: 1731025608774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:48,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:48,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:48,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:48,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:48,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50222 deadline: 1731025608776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:48,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:48,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:48,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:48,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:48,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1731025608777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:48,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:48,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:48,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:48,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:48,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50280 deadline: 1731025608887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:48,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:48,893 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:48,893 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:48,893 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:48,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50248 deadline: 1731025608887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:48,893 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:48,893 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:48,893 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:48,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:48,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50194 deadline: 1731025608887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:48,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:48,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:48,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:48,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:48,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50222 deadline: 1731025608897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:48,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:48,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:48,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1731025608898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:48,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:48,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:48,906 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:48,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:48,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:49,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:49,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50280 deadline: 1731025609097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:49,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:49,101 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:49,101 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:49,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50248 deadline: 1731025609097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:49,101 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:49,101 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:49,101 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:49,101 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:49,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:49,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50194 deadline: 1731025609107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:49,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:49,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:49,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:49,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:49,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50222 deadline: 1731025609117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:49,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:49,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:49,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1731025609117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:49,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:49,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:49,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:49,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:49,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:49,144 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/9eae889bf07a4633a8ab2fb9c695c310 2024-11-08T00:25:49,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/4348b5d1d0584f628ea659f4359a84e8 is 50, key is test_row_0/B:col10/1731025548729/Put/seqid=0 2024-11-08T00:25:49,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742343_1519 (size=12001) 2024-11-08T00:25:49,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:49,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50280 deadline: 1731025609407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:49,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:49,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50248 deadline: 1731025609407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:49,413 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:49,413 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:49,413 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:49,414 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:49,414 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:49,414 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:49,420 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T00:25:49,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:49,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50194 deadline: 1731025609417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:49,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:49,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:49,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:49,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:49,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50222 deadline: 1731025609428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:49,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:49,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:49,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:49,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1731025609428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:49,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:49,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:49,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:49,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:49,560 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/4348b5d1d0584f628ea659f4359a84e8 2024-11-08T00:25:49,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/80c1d6450cf544158213f63cc6c74e0a is 50, key is test_row_0/C:col10/1731025548729/Put/seqid=0 2024-11-08T00:25:49,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742344_1520 (size=12001) 2024-11-08T00:25:49,573 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/80c1d6450cf544158213f63cc6c74e0a 2024-11-08T00:25:49,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/9eae889bf07a4633a8ab2fb9c695c310 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/9eae889bf07a4633a8ab2fb9c695c310 2024-11-08T00:25:49,579 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/9eae889bf07a4633a8ab2fb9c695c310, entries=300, sequenceid=49, filesize=18.6 K 2024-11-08T00:25:49,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/4348b5d1d0584f628ea659f4359a84e8 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/4348b5d1d0584f628ea659f4359a84e8 2024-11-08T00:25:49,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/4348b5d1d0584f628ea659f4359a84e8, entries=150, sequenceid=49, filesize=11.7 K 2024-11-08T00:25:49,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/80c1d6450cf544158213f63cc6c74e0a as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/80c1d6450cf544158213f63cc6c74e0a 2024-11-08T00:25:49,588 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/80c1d6450cf544158213f63cc6c74e0a, entries=150, sequenceid=49, filesize=11.7 K 2024-11-08T00:25:49,589 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 04acb16af9bbc0bb9b137adb2146a06b in 858ms, sequenceid=49, compaction requested=true 2024-11-08T00:25:49,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 04acb16af9bbc0bb9b137adb2146a06b: 2024-11-08T00:25:49,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04acb16af9bbc0bb9b137adb2146a06b:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T00:25:49,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:25:49,589 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:25:49,589 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:25:49,590 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:25:49,590 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): 04acb16af9bbc0bb9b137adb2146a06b/A is initiating minor compaction (all files) 2024-11-08T00:25:49,590 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04acb16af9bbc0bb9b137adb2146a06b/A in TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:49,590 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/e407d50c85d64bfc847fa9e4a48292a1, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/510ec65edb8848b0908a03c84a718e32, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/9eae889bf07a4633a8ab2fb9c695c310] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp, totalSize=42.0 K 2024-11-08T00:25:49,590 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting e407d50c85d64bfc847fa9e4a48292a1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731025546486 2024-11-08T00:25:49,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04acb16af9bbc0bb9b137adb2146a06b:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T00:25:49,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:25:49,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04acb16af9bbc0bb9b137adb2146a06b:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:25:49,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:25:49,590 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 510ec65edb8848b0908a03c84a718e32, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731025546551 2024-11-08T00:25:49,591 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9eae889bf07a4633a8ab2fb9c695c310, keycount=300, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731025548722 2024-11-08T00:25:49,591 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:25:49,591 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): 04acb16af9bbc0bb9b137adb2146a06b/B is initiating minor compaction (all files) 2024-11-08T00:25:49,591 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04acb16af9bbc0bb9b137adb2146a06b/B in TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:49,591 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/c30363cdfa1d4f61a2486afc9809abb6, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/72b4450e2a2148b3a9f6e4f3c8750ce4, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/4348b5d1d0584f628ea659f4359a84e8] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp, totalSize=35.2 K 2024-11-08T00:25:49,592 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting c30363cdfa1d4f61a2486afc9809abb6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731025546486 2024-11-08T00:25:49,592 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 72b4450e2a2148b3a9f6e4f3c8750ce4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731025546551 2024-11-08T00:25:49,593 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 4348b5d1d0584f628ea659f4359a84e8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731025548729 2024-11-08T00:25:49,602 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04acb16af9bbc0bb9b137adb2146a06b#B#compaction#433 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:25:49,602 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/aea9d526c6e24da4ba58189cd4ef8f39 is 50, key is test_row_0/B:col10/1731025548729/Put/seqid=0 2024-11-08T00:25:49,605 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04acb16af9bbc0bb9b137adb2146a06b#A#compaction#434 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:25:49,605 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/944a138753394559981bcaa7a0ba48f1 is 50, key is test_row_0/A:col10/1731025548729/Put/seqid=0 2024-11-08T00:25:49,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742346_1522 (size=12104) 2024-11-08T00:25:49,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742345_1521 (size=12104) 2024-11-08T00:25:49,625 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/944a138753394559981bcaa7a0ba48f1 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/944a138753394559981bcaa7a0ba48f1 2024-11-08T00:25:49,628 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 04acb16af9bbc0bb9b137adb2146a06b/A of 04acb16af9bbc0bb9b137adb2146a06b into 944a138753394559981bcaa7a0ba48f1(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:25:49,628 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04acb16af9bbc0bb9b137adb2146a06b: 2024-11-08T00:25:49,628 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., storeName=04acb16af9bbc0bb9b137adb2146a06b/A, priority=13, startTime=1731025549589; duration=0sec 2024-11-08T00:25:49,628 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:25:49,629 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04acb16af9bbc0bb9b137adb2146a06b:A 2024-11-08T00:25:49,629 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:25:49,629 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:25:49,629 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): 04acb16af9bbc0bb9b137adb2146a06b/C is initiating minor compaction (all files) 2024-11-08T00:25:49,629 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04acb16af9bbc0bb9b137adb2146a06b/C in TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:49,629 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/66c27b728d084ebda6ad9c8577f694f7, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/d8f7538b6cba497f9b87a374193e04ec, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/80c1d6450cf544158213f63cc6c74e0a] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp, totalSize=35.2 K 2024-11-08T00:25:49,630 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66c27b728d084ebda6ad9c8577f694f7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731025546486 2024-11-08T00:25:49,630 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8f7538b6cba497f9b87a374193e04ec, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731025546551 2024-11-08T00:25:49,631 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80c1d6450cf544158213f63cc6c74e0a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731025548729 2024-11-08T00:25:49,637 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04acb16af9bbc0bb9b137adb2146a06b#C#compaction#435 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:25:49,637 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/e17c34434ee3451fa139e1b2f5d6b330 is 50, key is test_row_0/C:col10/1731025548729/Put/seqid=0 2024-11-08T00:25:49,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742347_1523 (size=12104) 2024-11-08T00:25:49,651 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/e17c34434ee3451fa139e1b2f5d6b330 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/e17c34434ee3451fa139e1b2f5d6b330 2024-11-08T00:25:49,656 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 04acb16af9bbc0bb9b137adb2146a06b/C of 04acb16af9bbc0bb9b137adb2146a06b into e17c34434ee3451fa139e1b2f5d6b330(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:25:49,656 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04acb16af9bbc0bb9b137adb2146a06b: 2024-11-08T00:25:49,656 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., storeName=04acb16af9bbc0bb9b137adb2146a06b/C, priority=13, startTime=1731025549590; duration=0sec 2024-11-08T00:25:49,656 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:25:49,656 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04acb16af9bbc0bb9b137adb2146a06b:C 2024-11-08T00:25:49,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 04acb16af9bbc0bb9b137adb2146a06b 2024-11-08T00:25:49,930 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:25:49,930 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 04acb16af9bbc0bb9b137adb2146a06b 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-08T00:25:49,935 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/02a59da620bf49208320f8d718f6655f is 50, key is test_row_0/A:col10/1731025549929/Put/seqid=0 2024-11-08T00:25:49,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742348_1524 (size=14341) 2024-11-08T00:25:49,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:49,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50280 deadline: 1731025609944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:49,950 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:49,950 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:49,950 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:49,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:49,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50222 deadline: 1731025609947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:49,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:49,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:49,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:49,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:49,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1731025609947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:49,955 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:49,955 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:49,955 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:49,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:49,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50248 deadline: 1731025609949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:49,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:49,957 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:49,957 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:49,957 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:49,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50194 deadline: 1731025609950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:49,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:49,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:49,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:50,022 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/aea9d526c6e24da4ba58189cd4ef8f39 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/aea9d526c6e24da4ba58189cd4ef8f39 2024-11-08T00:25:50,026 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 04acb16af9bbc0bb9b137adb2146a06b/B of 04acb16af9bbc0bb9b137adb2146a06b into aea9d526c6e24da4ba58189cd4ef8f39(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:25:50,026 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04acb16af9bbc0bb9b137adb2146a06b: 2024-11-08T00:25:50,026 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., storeName=04acb16af9bbc0bb9b137adb2146a06b/B, priority=13, startTime=1731025549589; duration=0sec 2024-11-08T00:25:50,026 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:25:50,026 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04acb16af9bbc0bb9b137adb2146a06b:B 2024-11-08T00:25:50,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:50,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50280 deadline: 1731025610057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:50,061 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:50,061 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:50,061 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:50,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:50,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50248 deadline: 1731025610067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:50,071 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:50,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:50,071 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:50,071 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:50,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50194 deadline: 1731025610067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:50,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:50,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:50,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:50,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:50,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50280 deadline: 1731025610267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:50,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:50,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:50,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:50,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:50,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50248 deadline: 1731025610277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:50,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:50,281 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:50,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50194 deadline: 1731025610277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:50,281 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:50,281 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:50,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:50,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:50,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:50,340 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/02a59da620bf49208320f8d718f6655f 2024-11-08T00:25:50,347 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/93511ef195f849d89cc01fadf679b261 is 50, key is test_row_0/B:col10/1731025549929/Put/seqid=0 2024-11-08T00:25:50,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742349_1525 (size=12001) 2024-11-08T00:25:50,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:50,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50280 deadline: 1731025610577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:50,578 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:50,578 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:50,578 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:50,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:50,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50248 deadline: 1731025610587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:50,595 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:50,595 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:50,595 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:50,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:50,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50194 deadline: 1731025610587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:50,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:50,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:50,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:50,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-08T00:25:50,647 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:25:50,648 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-08T00:25:50,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-08T00:25:50,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-08T00:25:50,649 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T00:25:50,650 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T00:25:50,650 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T00:25:50,750 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/93511ef195f849d89cc01fadf679b261 2024-11-08T00:25:50,756 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/db94ffb1f6dc4008884a8e9100938ee3 is 50, key is test_row_0/C:col10/1731025549929/Put/seqid=0 2024-11-08T00:25:50,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-08T00:25:50,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742350_1526 (size=12001) 2024-11-08T00:25:50,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-08T00:25:50,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:50,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. as already flushing 2024-11-08T00:25:50,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:50,802 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:50,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:50,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:50,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-08T00:25:50,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:50,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. as already flushing 2024-11-08T00:25:50,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:50,954 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:50,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:50,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:50,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-08T00:25:50,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:50,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50222 deadline: 1731025610967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:50,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:50,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1731025610968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:50,972 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 3302f0f507bd,40479,1731025415856 for 'test_row_1' in 04acb16af9bbc0bb9b137adb2146a06b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T00:25:50,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:50,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:50,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:50,973 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 3302f0f507bd,40479,1731025415856 for 'test_row_1' in 04acb16af9bbc0bb9b137adb2146a06b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T00:25:50,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:50,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:50,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:51,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:51,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50280 deadline: 1731025611086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:51,091 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:51,091 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:51,091 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:51,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-08T00:25:51,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:51,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. as already flushing 2024-11-08T00:25:51,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:51,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:51,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:51,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:51,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:51,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50248 deadline: 1731025611107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:51,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:51,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:51,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50194 deadline: 1731025611107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:51,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:51,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:51,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:51,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:51,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:51,162 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/db94ffb1f6dc4008884a8e9100938ee3 2024-11-08T00:25:51,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/02a59da620bf49208320f8d718f6655f as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/02a59da620bf49208320f8d718f6655f 2024-11-08T00:25:51,168 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/02a59da620bf49208320f8d718f6655f, entries=200, sequenceid=76, filesize=14.0 K 2024-11-08T00:25:51,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/93511ef195f849d89cc01fadf679b261 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/93511ef195f849d89cc01fadf679b261 2024-11-08T00:25:51,171 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/93511ef195f849d89cc01fadf679b261, entries=150, sequenceid=76, filesize=11.7 K 2024-11-08T00:25:51,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/db94ffb1f6dc4008884a8e9100938ee3 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/db94ffb1f6dc4008884a8e9100938ee3 2024-11-08T00:25:51,174 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/db94ffb1f6dc4008884a8e9100938ee3, entries=150, sequenceid=76, filesize=11.7 K 2024-11-08T00:25:51,175 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 04acb16af9bbc0bb9b137adb2146a06b in 1245ms, sequenceid=76, compaction requested=false 2024-11-08T00:25:51,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 04acb16af9bbc0bb9b137adb2146a06b: 2024-11-08T00:25:51,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-08T00:25:51,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:51,260 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 04acb16af9bbc0bb9b137adb2146a06b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T00:25:51,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/2a3ce38c06b1498890fc0ba76a3c7f81 is 50, key is test_row_0/A:col10/1731025549940/Put/seqid=0 2024-11-08T00:25:51,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742351_1527 (size=12001) 2024-11-08T00:25:51,266 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/2a3ce38c06b1498890fc0ba76a3c7f81 2024-11-08T00:25:51,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/6e2232c58b61456ebd8320af0eeb4a69 is 50, key is test_row_0/B:col10/1731025549940/Put/seqid=0 2024-11-08T00:25:51,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742352_1528 (size=12001) 2024-11-08T00:25:51,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-08T00:25:51,675 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/6e2232c58b61456ebd8320af0eeb4a69 2024-11-08T00:25:51,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/ac8b221a70f14dfea79ec35c80b377d6 is 50, key is test_row_0/C:col10/1731025549940/Put/seqid=0 2024-11-08T00:25:51,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742353_1529 (size=12001) 2024-11-08T00:25:51,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-08T00:25:52,084 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/ac8b221a70f14dfea79ec35c80b377d6 2024-11-08T00:25:52,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/2a3ce38c06b1498890fc0ba76a3c7f81 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/2a3ce38c06b1498890fc0ba76a3c7f81 2024-11-08T00:25:52,090 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/2a3ce38c06b1498890fc0ba76a3c7f81, entries=150, sequenceid=88, filesize=11.7 K 2024-11-08T00:25:52,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-08T00:25:52,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/6e2232c58b61456ebd8320af0eeb4a69 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/6e2232c58b61456ebd8320af0eeb4a69 2024-11-08T00:25:52,094 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/6e2232c58b61456ebd8320af0eeb4a69, entries=150, sequenceid=88, filesize=11.7 K 2024-11-08T00:25:52,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/ac8b221a70f14dfea79ec35c80b377d6 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/ac8b221a70f14dfea79ec35c80b377d6 2024-11-08T00:25:52,098 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/ac8b221a70f14dfea79ec35c80b377d6, entries=150, sequenceid=88, filesize=11.7 K 2024-11-08T00:25:52,099 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=6.71 KB/6870 for 04acb16af9bbc0bb9b137adb2146a06b in 840ms, sequenceid=88, compaction requested=true 2024-11-08T00:25:52,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 04acb16af9bbc0bb9b137adb2146a06b: 2024-11-08T00:25:52,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:52,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-08T00:25:52,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-08T00:25:52,102 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-08T00:25:52,102 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4510 sec 2024-11-08T00:25:52,104 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.4550 sec 2024-11-08T00:25:52,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 04acb16af9bbc0bb9b137adb2146a06b 2024-11-08T00:25:52,122 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:25:52,122 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 04acb16af9bbc0bb9b137adb2146a06b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T00:25:52,127 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/2a91b4ec4cca482ebb42fd165ab957da is 50, key is test_row_0/A:col10/1731025552121/Put/seqid=0 2024-11-08T00:25:52,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742354_1530 (size=14337) 2024-11-08T00:25:52,130 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/2a91b4ec4cca482ebb42fd165ab957da 2024-11-08T00:25:52,135 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/f68eb3ef3f34466aa8787e5dfaa879ff is 50, key is test_row_0/B:col10/1731025552121/Put/seqid=0 2024-11-08T00:25:52,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742355_1531 (size=9657) 2024-11-08T00:25:52,141 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/f68eb3ef3f34466aa8787e5dfaa879ff 2024-11-08T00:25:52,147 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/87e3b2c9b7244c29bfcf68e4c75f6894 is 50, key is test_row_0/C:col10/1731025552121/Put/seqid=0 2024-11-08T00:25:52,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742356_1532 (size=9657) 2024-11-08T00:25:52,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:52,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50280 deadline: 1731025612182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:52,193 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:52,193 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:52,193 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:52,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:52,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50194 deadline: 1731025612189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:52,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:52,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:52,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:52,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:52,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50248 deadline: 1731025612190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:52,199 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:52,199 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:52,199 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:52,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:52,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50280 deadline: 1731025612297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:52,302 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:52,302 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:52,302 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:52,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:52,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50194 deadline: 1731025612307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:52,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:52,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:52,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50248 deadline: 1731025612307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:52,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:52,314 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:52,314 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:52,314 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:52,314 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:52,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:52,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50280 deadline: 1731025612507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:52,512 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:52,512 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:52,512 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:52,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:52,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50194 deadline: 1731025612517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:52,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:52,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:52,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:52,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50248 deadline: 1731025612517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:52,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:52,521 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:52,521 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:52,521 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:52,550 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/87e3b2c9b7244c29bfcf68e4c75f6894 2024-11-08T00:25:52,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/2a91b4ec4cca482ebb42fd165ab957da as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/2a91b4ec4cca482ebb42fd165ab957da 2024-11-08T00:25:52,556 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/2a91b4ec4cca482ebb42fd165ab957da, entries=200, sequenceid=99, filesize=14.0 K 2024-11-08T00:25:52,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/f68eb3ef3f34466aa8787e5dfaa879ff as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/f68eb3ef3f34466aa8787e5dfaa879ff 2024-11-08T00:25:52,559 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/f68eb3ef3f34466aa8787e5dfaa879ff, entries=100, sequenceid=99, filesize=9.4 K 2024-11-08T00:25:52,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/87e3b2c9b7244c29bfcf68e4c75f6894 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/87e3b2c9b7244c29bfcf68e4c75f6894 2024-11-08T00:25:52,562 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/87e3b2c9b7244c29bfcf68e4c75f6894, entries=100, sequenceid=99, filesize=9.4 K 2024-11-08T00:25:52,563 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 04acb16af9bbc0bb9b137adb2146a06b in 441ms, sequenceid=99, compaction requested=true 2024-11-08T00:25:52,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 04acb16af9bbc0bb9b137adb2146a06b: 2024-11-08T00:25:52,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04acb16af9bbc0bb9b137adb2146a06b:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T00:25:52,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:25:52,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04acb16af9bbc0bb9b137adb2146a06b:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T00:25:52,563 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T00:25:52,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:25:52,563 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T00:25:52,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04acb16af9bbc0bb9b137adb2146a06b:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:25:52,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:25:52,564 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45763 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T00:25:52,564 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52783 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T00:25:52,564 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): 04acb16af9bbc0bb9b137adb2146a06b/A is initiating minor compaction (all files) 2024-11-08T00:25:52,564 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): 04acb16af9bbc0bb9b137adb2146a06b/B is initiating minor compaction (all files) 2024-11-08T00:25:52,564 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04acb16af9bbc0bb9b137adb2146a06b/A in TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:52,564 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04acb16af9bbc0bb9b137adb2146a06b/B in TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:52,564 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/aea9d526c6e24da4ba58189cd4ef8f39, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/93511ef195f849d89cc01fadf679b261, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/6e2232c58b61456ebd8320af0eeb4a69, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/f68eb3ef3f34466aa8787e5dfaa879ff] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp, totalSize=44.7 K 2024-11-08T00:25:52,564 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/944a138753394559981bcaa7a0ba48f1, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/02a59da620bf49208320f8d718f6655f, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/2a3ce38c06b1498890fc0ba76a3c7f81, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/2a91b4ec4cca482ebb42fd165ab957da] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp, totalSize=51.5 K 2024-11-08T00:25:52,564 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting aea9d526c6e24da4ba58189cd4ef8f39, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731025548729 2024-11-08T00:25:52,564 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 944a138753394559981bcaa7a0ba48f1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731025548729 2024-11-08T00:25:52,564 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 93511ef195f849d89cc01fadf679b261, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731025548774 2024-11-08T00:25:52,564 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02a59da620bf49208320f8d718f6655f, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731025548769 2024-11-08T00:25:52,565 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e2232c58b61456ebd8320af0eeb4a69, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731025549932 2024-11-08T00:25:52,565 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a3ce38c06b1498890fc0ba76a3c7f81, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731025549932 2024-11-08T00:25:52,565 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting f68eb3ef3f34466aa8787e5dfaa879ff, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731025552113 2024-11-08T00:25:52,565 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a91b4ec4cca482ebb42fd165ab957da, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731025552113 2024-11-08T00:25:52,572 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04acb16af9bbc0bb9b137adb2146a06b#A#compaction#446 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:25:52,572 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04acb16af9bbc0bb9b137adb2146a06b#B#compaction#445 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:25:52,572 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/a081f2cb07f2461a8a01366fd61acdb5 is 50, key is test_row_0/A:col10/1731025552121/Put/seqid=0 2024-11-08T00:25:52,573 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/5abd29aa3cb9484b9d1964042f8955cc is 50, key is test_row_0/B:col10/1731025552121/Put/seqid=0 2024-11-08T00:25:52,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742358_1534 (size=12241) 2024-11-08T00:25:52,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742357_1533 (size=12241) 2024-11-08T00:25:52,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-08T00:25:52,807 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:25:52,808 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-08T00:25:52,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-08T00:25:52,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-08T00:25:52,809 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T00:25:52,810 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T00:25:52,810 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T00:25:52,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 04acb16af9bbc0bb9b137adb2146a06b 2024-11-08T00:25:52,824 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:25:52,824 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 04acb16af9bbc0bb9b137adb2146a06b 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-08T00:25:52,829 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/0eabb0cffcd046f3975e1ebc51129a09 is 50, key is test_row_0/A:col10/1731025552189/Put/seqid=0 2024-11-08T00:25:52,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742359_1535 (size=14341) 2024-11-08T00:25:52,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:52,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50194 deadline: 1731025612838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:52,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:52,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50248 deadline: 1731025612838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:52,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:52,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:52,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:52,844 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:52,844 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:52,844 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:52,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:52,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50280 deadline: 1731025612844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:52,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:52,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:52,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:52,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-08T00:25:52,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:52,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50194 deadline: 1731025612947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:52,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:52,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:52,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:52,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:52,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50248 deadline: 1731025612947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:52,955 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:52,955 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:52,955 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:52,962 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-08T00:25:52,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:52,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. as already flushing 2024-11-08T00:25:52,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:52,962 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:52,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:52,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:52,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:52,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50280 deadline: 1731025612957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:52,965 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:52,965 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:52,965 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:52,980 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/5abd29aa3cb9484b9d1964042f8955cc as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/5abd29aa3cb9484b9d1964042f8955cc 2024-11-08T00:25:52,980 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/a081f2cb07f2461a8a01366fd61acdb5 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/a081f2cb07f2461a8a01366fd61acdb5 2024-11-08T00:25:52,985 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 04acb16af9bbc0bb9b137adb2146a06b/B of 04acb16af9bbc0bb9b137adb2146a06b into 5abd29aa3cb9484b9d1964042f8955cc(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:25:52,985 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04acb16af9bbc0bb9b137adb2146a06b: 2024-11-08T00:25:52,985 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., storeName=04acb16af9bbc0bb9b137adb2146a06b/B, priority=12, startTime=1731025552563; duration=0sec 2024-11-08T00:25:52,985 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 04acb16af9bbc0bb9b137adb2146a06b/A of 04acb16af9bbc0bb9b137adb2146a06b into a081f2cb07f2461a8a01366fd61acdb5(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:25:52,985 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04acb16af9bbc0bb9b137adb2146a06b: 2024-11-08T00:25:52,985 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., storeName=04acb16af9bbc0bb9b137adb2146a06b/A, priority=12, startTime=1731025552563; duration=0sec 2024-11-08T00:25:52,985 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:25:52,985 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04acb16af9bbc0bb9b137adb2146a06b:B 2024-11-08T00:25:52,985 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:25:52,985 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04acb16af9bbc0bb9b137adb2146a06b:A 2024-11-08T00:25:52,985 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T00:25:52,986 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45763 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T00:25:52,986 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): 04acb16af9bbc0bb9b137adb2146a06b/C is initiating minor compaction (all files) 2024-11-08T00:25:52,986 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04acb16af9bbc0bb9b137adb2146a06b/C in TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:52,986 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/e17c34434ee3451fa139e1b2f5d6b330, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/db94ffb1f6dc4008884a8e9100938ee3, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/ac8b221a70f14dfea79ec35c80b377d6, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/87e3b2c9b7244c29bfcf68e4c75f6894] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp, totalSize=44.7 K 2024-11-08T00:25:52,986 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting e17c34434ee3451fa139e1b2f5d6b330, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731025548729 2024-11-08T00:25:52,986 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting db94ffb1f6dc4008884a8e9100938ee3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731025548774 2024-11-08T00:25:52,986 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting ac8b221a70f14dfea79ec35c80b377d6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731025549932 2024-11-08T00:25:52,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:52,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50222 deadline: 1731025612977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:52,987 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 87e3b2c9b7244c29bfcf68e4c75f6894, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731025552113 2024-11-08T00:25:52,987 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 3302f0f507bd,40479,1731025415856 for 'test_row_1' in 04acb16af9bbc0bb9b137adb2146a06b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T00:25:52,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:52,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:52,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:52,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:52,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1731025612987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:52,990 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 3302f0f507bd,40479,1731025415856 for 'test_row_1' in 04acb16af9bbc0bb9b137adb2146a06b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T00:25:52,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:52,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:52,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:52,996 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04acb16af9bbc0bb9b137adb2146a06b#C#compaction#448 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:25:52,996 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/927fb6fa175348f38547ea2f08049ccf is 50, key is test_row_0/C:col10/1731025552121/Put/seqid=0 2024-11-08T00:25:53,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742360_1536 (size=12241) 2024-11-08T00:25:53,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-08T00:25:53,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:53,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. as already flushing 2024-11-08T00:25:53,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:53,115 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:53,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:53,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:53,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-08T00:25:53,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:53,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50194 deadline: 1731025613157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:53,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:53,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:53,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:53,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:53,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50248 deadline: 1731025613158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:53,166 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:53,166 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:53,166 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:53,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:53,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50280 deadline: 1731025613177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:53,181 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:53,181 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:53,181 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:53,232 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/0eabb0cffcd046f3975e1ebc51129a09 2024-11-08T00:25:53,238 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/67f0e42e280d4ce1a2906cbe8583f33f is 50, key is test_row_0/B:col10/1731025552189/Put/seqid=0 2024-11-08T00:25:53,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742361_1537 (size=12001) 2024-11-08T00:25:53,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-08T00:25:53,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:53,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. as already flushing 2024-11-08T00:25:53,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:53,267 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:53,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:53,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:53,411 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/927fb6fa175348f38547ea2f08049ccf as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/927fb6fa175348f38547ea2f08049ccf 2024-11-08T00:25:53,414 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 04acb16af9bbc0bb9b137adb2146a06b/C of 04acb16af9bbc0bb9b137adb2146a06b into 927fb6fa175348f38547ea2f08049ccf(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:25:53,414 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04acb16af9bbc0bb9b137adb2146a06b: 2024-11-08T00:25:53,414 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., storeName=04acb16af9bbc0bb9b137adb2146a06b/C, priority=12, startTime=1731025552563; duration=0sec 2024-11-08T00:25:53,414 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:25:53,414 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04acb16af9bbc0bb9b137adb2146a06b:C 2024-11-08T00:25:53,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-08T00:25:53,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:53,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. as already flushing 2024-11-08T00:25:53,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:53,420 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:53,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:53,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:53,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-08T00:25:53,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:53,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50194 deadline: 1731025613477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:53,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:53,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50248 deadline: 1731025613477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:53,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:53,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:53,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:53,482 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:53,482 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:53,482 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:53,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:53,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50280 deadline: 1731025613487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:53,493 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:53,493 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:53,493 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:53,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-08T00:25:53,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:53,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. as already flushing 2024-11-08T00:25:53,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:53,573 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:53,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:53,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:53,643 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/67f0e42e280d4ce1a2906cbe8583f33f 2024-11-08T00:25:53,648 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/9e2b0c9399804c90835bb651b7a638c4 is 50, key is test_row_0/C:col10/1731025552189/Put/seqid=0 2024-11-08T00:25:53,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742362_1538 (size=12001) 2024-11-08T00:25:53,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-08T00:25:53,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:53,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. as already flushing 2024-11-08T00:25:53,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:53,725 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:53,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:53,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:53,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-08T00:25:53,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:53,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. as already flushing 2024-11-08T00:25:53,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:53,878 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:53,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:53,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:53,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-08T00:25:53,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:53,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50194 deadline: 1731025613987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:53,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:53,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:53,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50248 deadline: 1731025613987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:53,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:53,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:53,990 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:53,990 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:53,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:54,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:54,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50280 deadline: 1731025614007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:54,013 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:54,013 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:54,013 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:54,030 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-08T00:25:54,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:54,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. as already flushing 2024-11-08T00:25:54,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:54,031 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:54,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:54,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:54,051 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/9e2b0c9399804c90835bb651b7a638c4 2024-11-08T00:25:54,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/0eabb0cffcd046f3975e1ebc51129a09 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/0eabb0cffcd046f3975e1ebc51129a09 2024-11-08T00:25:54,058 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/0eabb0cffcd046f3975e1ebc51129a09, entries=200, sequenceid=125, filesize=14.0 K 2024-11-08T00:25:54,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/67f0e42e280d4ce1a2906cbe8583f33f as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/67f0e42e280d4ce1a2906cbe8583f33f 2024-11-08T00:25:54,061 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/67f0e42e280d4ce1a2906cbe8583f33f, entries=150, sequenceid=125, filesize=11.7 K 2024-11-08T00:25:54,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/9e2b0c9399804c90835bb651b7a638c4 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/9e2b0c9399804c90835bb651b7a638c4 2024-11-08T00:25:54,070 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/9e2b0c9399804c90835bb651b7a638c4, entries=150, sequenceid=125, filesize=11.7 K 2024-11-08T00:25:54,071 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 04acb16af9bbc0bb9b137adb2146a06b in 1246ms, sequenceid=125, compaction requested=false 2024-11-08T00:25:54,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 04acb16af9bbc0bb9b137adb2146a06b: 2024-11-08T00:25:54,184 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-08T00:25:54,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:54,184 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 04acb16af9bbc0bb9b137adb2146a06b 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-08T00:25:54,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/5c7d1f7e98d142feb074c44b023f4207 is 50, key is test_row_0/A:col10/1731025552843/Put/seqid=0 2024-11-08T00:25:54,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742363_1539 (size=12101) 2024-11-08T00:25:54,591 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/5c7d1f7e98d142feb074c44b023f4207 2024-11-08T00:25:54,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/389e7bc65d0d40649e5316c891fe8a4e is 50, key is test_row_0/B:col10/1731025552843/Put/seqid=0 2024-11-08T00:25:54,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742364_1540 (size=12101) 2024-11-08T00:25:54,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-08T00:25:55,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. as already flushing 2024-11-08T00:25:55,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 04acb16af9bbc0bb9b137adb2146a06b 2024-11-08T00:25:55,017 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/389e7bc65d0d40649e5316c891fe8a4e 2024-11-08T00:25:55,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/a98c2637756d45d983157fd7dbe90dc6 is 50, key is test_row_0/C:col10/1731025552843/Put/seqid=0 2024-11-08T00:25:55,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742365_1541 (size=12101) 2024-11-08T00:25:55,030 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/a98c2637756d45d983157fd7dbe90dc6 2024-11-08T00:25:55,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/5c7d1f7e98d142feb074c44b023f4207 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/5c7d1f7e98d142feb074c44b023f4207 2024-11-08T00:25:55,039 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/5c7d1f7e98d142feb074c44b023f4207, entries=150, sequenceid=138, filesize=11.8 K 2024-11-08T00:25:55,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/389e7bc65d0d40649e5316c891fe8a4e as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/389e7bc65d0d40649e5316c891fe8a4e 2024-11-08T00:25:55,043 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/389e7bc65d0d40649e5316c891fe8a4e, entries=150, sequenceid=138, filesize=11.8 K 2024-11-08T00:25:55,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/a98c2637756d45d983157fd7dbe90dc6 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/a98c2637756d45d983157fd7dbe90dc6 2024-11-08T00:25:55,049 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/a98c2637756d45d983157fd7dbe90dc6, entries=150, sequenceid=138, filesize=11.8 K 2024-11-08T00:25:55,051 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=80.51 KB/82440 for 04acb16af9bbc0bb9b137adb2146a06b in 866ms, sequenceid=138, compaction requested=true 2024-11-08T00:25:55,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 04acb16af9bbc0bb9b137adb2146a06b 2024-11-08T00:25:55,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 04acb16af9bbc0bb9b137adb2146a06b: 2024-11-08T00:25:55,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:55,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-08T00:25:55,052 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:25:55,052 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 04acb16af9bbc0bb9b137adb2146a06b 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-08T00:25:55,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-08T00:25:55,053 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-08T00:25:55,053 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-08T00:25:55,055 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-08T00:25:55,055 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2430 sec 2024-11-08T00:25:55,059 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 2.2470 sec 2024-11-08T00:25:55,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/a761abec878842d3ac51f11995d54c99 is 50, key is test_row_0/A:col10/1731025555019/Put/seqid=0 2024-11-08T00:25:55,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742366_1542 (size=14541) 2024-11-08T00:25:55,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:55,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50194 deadline: 1731025615118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:55,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:55,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:55,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:55,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:55,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50280 deadline: 1731025615128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:55,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:55,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50248 deadline: 1731025615129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:55,139 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:55,139 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:55,139 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:55,139 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:55,139 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:55,139 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:55,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:55,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50194 deadline: 1731025615237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:55,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:55,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:55,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:55,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:55,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50280 deadline: 1731025615247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:55,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:55,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50248 deadline: 1731025615247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:55,259 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:55,259 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:55,259 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:55,259 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:55,259 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:55,259 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:55,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:55,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50194 deadline: 1731025615459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:55,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:55,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50280 deadline: 1731025615467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:55,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:55,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50248 deadline: 1731025615467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:55,469 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:55,469 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:55,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:55,469 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:55,469 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:55,469 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:55,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:55,469 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:55,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:55,474 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/a761abec878842d3ac51f11995d54c99 2024-11-08T00:25:55,479 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/54007788ab404ec7b77d737f7c2490da is 50, key is test_row_0/B:col10/1731025555019/Put/seqid=0 2024-11-08T00:25:55,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742367_1543 (size=12151) 2024-11-08T00:25:55,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:55,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50280 deadline: 1731025615777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:55,789 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:55,789 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:55,789 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:55,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:55,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50248 deadline: 1731025615777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:55,789 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:55,789 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:55,789 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:55,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:55,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50194 deadline: 1731025615777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:55,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:55,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:55,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:55,883 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/54007788ab404ec7b77d737f7c2490da 2024-11-08T00:25:55,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/110dcc6900f44949a53d3825bc885c2b is 50, key is test_row_0/C:col10/1731025555019/Put/seqid=0 2024-11-08T00:25:55,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742368_1544 (size=12151) 2024-11-08T00:25:56,297 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/110dcc6900f44949a53d3825bc885c2b 2024-11-08T00:25:56,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/a761abec878842d3ac51f11995d54c99 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/a761abec878842d3ac51f11995d54c99 2024-11-08T00:25:56,303 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/a761abec878842d3ac51f11995d54c99, entries=200, sequenceid=155, filesize=14.2 K 2024-11-08T00:25:56,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/54007788ab404ec7b77d737f7c2490da as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/54007788ab404ec7b77d737f7c2490da 2024-11-08T00:25:56,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:56,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50248 deadline: 1731025616298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:56,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:56,306 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:56,306 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:56,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50194 deadline: 1731025616298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:56,306 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:56,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:56,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:56,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:56,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:56,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50280 deadline: 1731025616298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:56,306 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:56,307 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:56,307 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:56,317 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/54007788ab404ec7b77d737f7c2490da, entries=150, sequenceid=155, filesize=11.9 K 2024-11-08T00:25:56,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/110dcc6900f44949a53d3825bc885c2b as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/110dcc6900f44949a53d3825bc885c2b 2024-11-08T00:25:56,321 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/110dcc6900f44949a53d3825bc885c2b, entries=150, sequenceid=155, filesize=11.9 K 2024-11-08T00:25:56,322 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 04acb16af9bbc0bb9b137adb2146a06b in 1270ms, sequenceid=155, compaction requested=true 2024-11-08T00:25:56,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 04acb16af9bbc0bb9b137adb2146a06b: 2024-11-08T00:25:56,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04acb16af9bbc0bb9b137adb2146a06b:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T00:25:56,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:25:56,322 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T00:25:56,322 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T00:25:56,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04acb16af9bbc0bb9b137adb2146a06b:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T00:25:56,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:25:56,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04acb16af9bbc0bb9b137adb2146a06b:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:25:56,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:25:56,323 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53224 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T00:25:56,323 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48494 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T00:25:56,323 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): 04acb16af9bbc0bb9b137adb2146a06b/B is initiating minor compaction (all files) 2024-11-08T00:25:56,323 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): 04acb16af9bbc0bb9b137adb2146a06b/A is initiating minor compaction (all files) 2024-11-08T00:25:56,323 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04acb16af9bbc0bb9b137adb2146a06b/B in TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:56,323 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04acb16af9bbc0bb9b137adb2146a06b/A in TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:56,323 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/5abd29aa3cb9484b9d1964042f8955cc, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/67f0e42e280d4ce1a2906cbe8583f33f, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/389e7bc65d0d40649e5316c891fe8a4e, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/54007788ab404ec7b77d737f7c2490da] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp, totalSize=47.4 K 2024-11-08T00:25:56,323 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/a081f2cb07f2461a8a01366fd61acdb5, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/0eabb0cffcd046f3975e1ebc51129a09, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/5c7d1f7e98d142feb074c44b023f4207, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/a761abec878842d3ac51f11995d54c99] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp, totalSize=52.0 K 2024-11-08T00:25:56,324 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 5abd29aa3cb9484b9d1964042f8955cc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731025549949 2024-11-08T00:25:56,324 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting a081f2cb07f2461a8a01366fd61acdb5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731025549949 2024-11-08T00:25:56,324 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0eabb0cffcd046f3975e1ebc51129a09, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1731025552179 2024-11-08T00:25:56,324 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 67f0e42e280d4ce1a2906cbe8583f33f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1731025552179 2024-11-08T00:25:56,324 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 389e7bc65d0d40649e5316c891fe8a4e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731025552828 2024-11-08T00:25:56,324 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c7d1f7e98d142feb074c44b023f4207, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731025552828 2024-11-08T00:25:56,324 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 54007788ab404ec7b77d737f7c2490da, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731025555019 2024-11-08T00:25:56,324 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting a761abec878842d3ac51f11995d54c99, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731025555019 2024-11-08T00:25:56,331 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04acb16af9bbc0bb9b137adb2146a06b#A#compaction#457 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:25:56,331 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/6fb43869503842eb8db07ac146bc5b51 is 50, key is test_row_0/A:col10/1731025555019/Put/seqid=0 2024-11-08T00:25:56,333 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04acb16af9bbc0bb9b137adb2146a06b#B#compaction#458 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:25:56,333 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/702cd2145f494d1887469204028ac6f7 is 50, key is test_row_0/B:col10/1731025555019/Put/seqid=0 2024-11-08T00:25:56,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742369_1545 (size=12527) 2024-11-08T00:25:56,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742370_1546 (size=12527) 2024-11-08T00:25:56,352 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/702cd2145f494d1887469204028ac6f7 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/702cd2145f494d1887469204028ac6f7 2024-11-08T00:25:56,359 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 04acb16af9bbc0bb9b137adb2146a06b/B of 04acb16af9bbc0bb9b137adb2146a06b into 702cd2145f494d1887469204028ac6f7(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:25:56,359 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04acb16af9bbc0bb9b137adb2146a06b: 2024-11-08T00:25:56,359 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., storeName=04acb16af9bbc0bb9b137adb2146a06b/B, priority=12, startTime=1731025556322; duration=0sec 2024-11-08T00:25:56,359 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:25:56,359 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04acb16af9bbc0bb9b137adb2146a06b:B 2024-11-08T00:25:56,359 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T00:25:56,360 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48494 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T00:25:56,360 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): 04acb16af9bbc0bb9b137adb2146a06b/C is initiating minor compaction (all files) 2024-11-08T00:25:56,360 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04acb16af9bbc0bb9b137adb2146a06b/C in TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:56,360 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/927fb6fa175348f38547ea2f08049ccf, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/9e2b0c9399804c90835bb651b7a638c4, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/a98c2637756d45d983157fd7dbe90dc6, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/110dcc6900f44949a53d3825bc885c2b] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp, totalSize=47.4 K 2024-11-08T00:25:56,360 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 927fb6fa175348f38547ea2f08049ccf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731025549949 2024-11-08T00:25:56,361 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e2b0c9399804c90835bb651b7a638c4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1731025552179 2024-11-08T00:25:56,361 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting a98c2637756d45d983157fd7dbe90dc6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731025552828 2024-11-08T00:25:56,361 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 110dcc6900f44949a53d3825bc885c2b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731025555019 2024-11-08T00:25:56,367 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04acb16af9bbc0bb9b137adb2146a06b#C#compaction#459 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:25:56,368 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/94d5cd355f3f49a1aee9ac9af6b1adae is 50, key is test_row_0/C:col10/1731025555019/Put/seqid=0 2024-11-08T00:25:56,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742371_1547 (size=12527) 2024-11-08T00:25:56,745 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/6fb43869503842eb8db07ac146bc5b51 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/6fb43869503842eb8db07ac146bc5b51 2024-11-08T00:25:56,749 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 04acb16af9bbc0bb9b137adb2146a06b/A of 04acb16af9bbc0bb9b137adb2146a06b into 6fb43869503842eb8db07ac146bc5b51(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:25:56,749 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04acb16af9bbc0bb9b137adb2146a06b: 2024-11-08T00:25:56,749 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., storeName=04acb16af9bbc0bb9b137adb2146a06b/A, priority=12, startTime=1731025556322; duration=0sec 2024-11-08T00:25:56,749 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:25:56,749 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04acb16af9bbc0bb9b137adb2146a06b:A 2024-11-08T00:25:56,775 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/94d5cd355f3f49a1aee9ac9af6b1adae as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/94d5cd355f3f49a1aee9ac9af6b1adae 2024-11-08T00:25:56,778 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 04acb16af9bbc0bb9b137adb2146a06b/C of 04acb16af9bbc0bb9b137adb2146a06b into 94d5cd355f3f49a1aee9ac9af6b1adae(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:25:56,778 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04acb16af9bbc0bb9b137adb2146a06b: 2024-11-08T00:25:56,778 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., storeName=04acb16af9bbc0bb9b137adb2146a06b/C, priority=12, startTime=1731025556322; duration=0sec 2024-11-08T00:25:56,778 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:25:56,779 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04acb16af9bbc0bb9b137adb2146a06b:C 2024-11-08T00:25:56,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-08T00:25:56,967 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:25:56,967 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-08T00:25:56,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-08T00:25:56,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-08T00:25:56,969 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T00:25:56,969 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T00:25:56,970 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T00:25:57,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 04acb16af9bbc0bb9b137adb2146a06b 2024-11-08T00:25:57,031 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:25:57,031 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 04acb16af9bbc0bb9b137adb2146a06b 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-08T00:25:57,035 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/21609a4d5154401f99ebb9986fbc1025 is 50, key is test_row_0/A:col10/1731025555127/Put/seqid=0 2024-11-08T00:25:57,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742372_1548 (size=14541) 2024-11-08T00:25:57,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-08T00:25:57,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:57,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1731025617079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:57,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:57,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:57,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:57,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:57,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50222 deadline: 1731025617080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:57,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:57,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:57,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:57,121 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-08T00:25:57,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:57,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. as already flushing 2024-11-08T00:25:57,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:57,122 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:57,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:57,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:57,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:57,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1731025617187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:57,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:57,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:57,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:57,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:57,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50222 deadline: 1731025617197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:57,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:57,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:57,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:57,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-08T00:25:57,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:57,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. as already flushing 2024-11-08T00:25:57,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:57,274 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:57,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:57,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:57,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-08T00:25:57,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:57,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50248 deadline: 1731025617317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:57,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:57,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50194 deadline: 1731025617317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:57,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:57,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50280 deadline: 1731025617317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:57,323 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 3302f0f507bd,40479,1731025415856 for 'test_row_2' in 04acb16af9bbc0bb9b137adb2146a06b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T00:25:57,324 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:57,324 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:57,324 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 3302f0f507bd,40479,1731025415856 for 'test_row_0' in 04acb16af9bbc0bb9b137adb2146a06b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T00:25:57,324 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:57,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:57,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:57,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:57,325 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 3302f0f507bd,40479,1731025415856 for 'test_row_2' in 04acb16af9bbc0bb9b137adb2146a06b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T00:25:57,325 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:57,325 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:57,325 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:57,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:57,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1731025617408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:57,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:57,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:57,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50222 deadline: 1731025617408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:57,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:57,413 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:57,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:57,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:57,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:57,428 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-08T00:25:57,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:57,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. as already flushing 2024-11-08T00:25:57,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:57,428 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:57,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:57,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:57,453 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/21609a4d5154401f99ebb9986fbc1025 2024-11-08T00:25:57,458 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/a22a245055394828beab86a6a6680d61 is 50, key is test_row_0/B:col10/1731025555127/Put/seqid=0 2024-11-08T00:25:57,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742373_1549 (size=12151) 2024-11-08T00:25:57,461 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/a22a245055394828beab86a6a6680d61 2024-11-08T00:25:57,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/abd1d452fc2146fca43432fa4d820585 is 50, key is test_row_0/C:col10/1731025555127/Put/seqid=0 2024-11-08T00:25:57,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742374_1550 (size=12151) 2024-11-08T00:25:57,474 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/abd1d452fc2146fca43432fa4d820585 2024-11-08T00:25:57,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/21609a4d5154401f99ebb9986fbc1025 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/21609a4d5154401f99ebb9986fbc1025 2024-11-08T00:25:57,487 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/21609a4d5154401f99ebb9986fbc1025, entries=200, sequenceid=178, filesize=14.2 K 2024-11-08T00:25:57,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/a22a245055394828beab86a6a6680d61 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/a22a245055394828beab86a6a6680d61 2024-11-08T00:25:57,494 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/a22a245055394828beab86a6a6680d61, entries=150, sequenceid=178, filesize=11.9 K 2024-11-08T00:25:57,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/abd1d452fc2146fca43432fa4d820585 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/abd1d452fc2146fca43432fa4d820585 2024-11-08T00:25:57,501 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/abd1d452fc2146fca43432fa4d820585, entries=150, sequenceid=178, filesize=11.9 K 2024-11-08T00:25:57,501 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 04acb16af9bbc0bb9b137adb2146a06b in 470ms, sequenceid=178, compaction requested=false 2024-11-08T00:25:57,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 04acb16af9bbc0bb9b137adb2146a06b: 2024-11-08T00:25:57,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-08T00:25:57,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:57,581 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 04acb16af9bbc0bb9b137adb2146a06b 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-08T00:25:57,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/659b74e8c28f4378b878701e6e9174d2 is 50, key is test_row_0/A:col10/1731025557079/Put/seqid=0 2024-11-08T00:25:57,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742375_1551 (size=12151) 2024-11-08T00:25:57,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-08T00:25:57,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 04acb16af9bbc0bb9b137adb2146a06b 2024-11-08T00:25:57,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. as already flushing 2024-11-08T00:25:57,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:57,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50222 deadline: 1731025617789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:57,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:57,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:57,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:57,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:57,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1731025617796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:57,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:57,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:57,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:57,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:57,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50222 deadline: 1731025617907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:57,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:57,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:57,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:57,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:57,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1731025617917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:57,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:57,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:57,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:57,990 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/659b74e8c28f4378b878701e6e9174d2 2024-11-08T00:25:57,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/a585705bff044a5381f1db737f0b3ca1 is 50, key is test_row_0/B:col10/1731025557079/Put/seqid=0 2024-11-08T00:25:57,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742376_1552 (size=12151) 2024-11-08T00:25:58,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-08T00:25:58,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:58,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50222 deadline: 1731025618127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:58,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:58,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1731025618127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:58,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:58,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:58,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:58,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:58,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:58,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:58,399 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/a585705bff044a5381f1db737f0b3ca1 2024-11-08T00:25:58,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/f57a2f514e304a7db7624dabcd63c49a is 50, key is test_row_0/C:col10/1731025557079/Put/seqid=0 2024-11-08T00:25:58,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742377_1553 (size=12151) 2024-11-08T00:25:58,407 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/f57a2f514e304a7db7624dabcd63c49a 2024-11-08T00:25:58,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/659b74e8c28f4378b878701e6e9174d2 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/659b74e8c28f4378b878701e6e9174d2 2024-11-08T00:25:58,417 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/659b74e8c28f4378b878701e6e9174d2, entries=150, sequenceid=194, filesize=11.9 K 2024-11-08T00:25:58,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/a585705bff044a5381f1db737f0b3ca1 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/a585705bff044a5381f1db737f0b3ca1 2024-11-08T00:25:58,421 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/a585705bff044a5381f1db737f0b3ca1, entries=150, sequenceid=194, filesize=11.9 K 2024-11-08T00:25:58,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/f57a2f514e304a7db7624dabcd63c49a as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/f57a2f514e304a7db7624dabcd63c49a 2024-11-08T00:25:58,424 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/f57a2f514e304a7db7624dabcd63c49a, entries=150, sequenceid=194, filesize=11.9 K 2024-11-08T00:25:58,425 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 04acb16af9bbc0bb9b137adb2146a06b in 844ms, sequenceid=194, compaction requested=true 2024-11-08T00:25:58,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 04acb16af9bbc0bb9b137adb2146a06b: 2024-11-08T00:25:58,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:58,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-08T00:25:58,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-08T00:25:58,427 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-08T00:25:58,427 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4560 sec 2024-11-08T00:25:58,429 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.4610 sec 2024-11-08T00:25:58,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 04acb16af9bbc0bb9b137adb2146a06b 2024-11-08T00:25:58,440 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:25:58,440 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 04acb16af9bbc0bb9b137adb2146a06b 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-08T00:25:58,443 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/aa290aea6aa343d88c1bd42f4932955e is 50, key is test_row_0/A:col10/1731025558439/Put/seqid=0 2024-11-08T00:25:58,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742378_1554 (size=14541) 2024-11-08T00:25:58,446 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/aa290aea6aa343d88c1bd42f4932955e 2024-11-08T00:25:58,452 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/bcd05fd7ab314f158b2cccbd3c3d02ac is 50, key is test_row_0/B:col10/1731025558439/Put/seqid=0 2024-11-08T00:25:58,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742379_1555 (size=12151) 2024-11-08T00:25:58,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:58,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50222 deadline: 1731025618510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:58,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:58,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:58,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:58,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:58,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1731025618519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:58,523 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:58,523 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:58,523 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:58,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:58,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50222 deadline: 1731025618628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:58,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:58,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1731025618628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:58,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:58,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:58,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:58,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:58,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:58,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:58,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:58,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50222 deadline: 1731025618837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:58,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:58,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1731025618837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:58,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:58,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:58,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:58,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:58,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:58,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:58,856 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/bcd05fd7ab314f158b2cccbd3c3d02ac 2024-11-08T00:25:58,861 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/8bbd663759b54b8ab21d59af15e1557b is 50, key is test_row_0/C:col10/1731025558439/Put/seqid=0 2024-11-08T00:25:58,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742380_1556 (size=12151) 2024-11-08T00:25:59,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-08T00:25:59,117 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:25:59,118 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-08T00:25:59,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-08T00:25:59,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-08T00:25:59,119 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T00:25:59,120 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T00:25:59,120 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T00:25:59,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:59,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50222 deadline: 1731025619157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:59,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:59,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:59,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:59,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1731025619157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:59,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:59,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:59,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:59,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:59,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-08T00:25:59,264 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/8bbd663759b54b8ab21d59af15e1557b 2024-11-08T00:25:59,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/aa290aea6aa343d88c1bd42f4932955e as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/aa290aea6aa343d88c1bd42f4932955e 2024-11-08T00:25:59,270 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/aa290aea6aa343d88c1bd42f4932955e, entries=200, sequenceid=215, filesize=14.2 K 2024-11-08T00:25:59,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/bcd05fd7ab314f158b2cccbd3c3d02ac as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/bcd05fd7ab314f158b2cccbd3c3d02ac 2024-11-08T00:25:59,273 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/bcd05fd7ab314f158b2cccbd3c3d02ac, entries=150, sequenceid=215, filesize=11.9 K 2024-11-08T00:25:59,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-08T00:25:59,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:59,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. as already flushing 2024-11-08T00:25:59,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:59,274 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:59,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:59,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:59,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/8bbd663759b54b8ab21d59af15e1557b as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/8bbd663759b54b8ab21d59af15e1557b 2024-11-08T00:25:59,279 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/8bbd663759b54b8ab21d59af15e1557b, entries=150, sequenceid=215, filesize=11.9 K 2024-11-08T00:25:59,280 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 04acb16af9bbc0bb9b137adb2146a06b in 840ms, sequenceid=215, compaction requested=true 2024-11-08T00:25:59,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 04acb16af9bbc0bb9b137adb2146a06b: 2024-11-08T00:25:59,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04acb16af9bbc0bb9b137adb2146a06b:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T00:25:59,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:25:59,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04acb16af9bbc0bb9b137adb2146a06b:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T00:25:59,280 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T00:25:59,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:25:59,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04acb16af9bbc0bb9b137adb2146a06b:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:25:59,280 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T00:25:59,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:25:59,284 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T00:25:59,284 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53760 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T00:25:59,284 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): 04acb16af9bbc0bb9b137adb2146a06b/B is initiating minor compaction (all files) 2024-11-08T00:25:59,284 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): 04acb16af9bbc0bb9b137adb2146a06b/A is initiating minor compaction (all files) 2024-11-08T00:25:59,284 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04acb16af9bbc0bb9b137adb2146a06b/B in TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:59,284 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04acb16af9bbc0bb9b137adb2146a06b/A in TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:59,284 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/702cd2145f494d1887469204028ac6f7, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/a22a245055394828beab86a6a6680d61, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/a585705bff044a5381f1db737f0b3ca1, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/bcd05fd7ab314f158b2cccbd3c3d02ac] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp, totalSize=47.8 K 2024-11-08T00:25:59,284 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/6fb43869503842eb8db07ac146bc5b51, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/21609a4d5154401f99ebb9986fbc1025, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/659b74e8c28f4378b878701e6e9174d2, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/aa290aea6aa343d88c1bd42f4932955e] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp, totalSize=52.5 K 2024-11-08T00:25:59,284 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 702cd2145f494d1887469204028ac6f7, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731025555019 2024-11-08T00:25:59,284 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6fb43869503842eb8db07ac146bc5b51, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731025555019 2024-11-08T00:25:59,284 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting a22a245055394828beab86a6a6680d61, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731025555115 2024-11-08T00:25:59,284 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21609a4d5154401f99ebb9986fbc1025, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731025555115 2024-11-08T00:25:59,284 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting a585705bff044a5381f1db737f0b3ca1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731025557055 2024-11-08T00:25:59,284 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 659b74e8c28f4378b878701e6e9174d2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731025557055 2024-11-08T00:25:59,285 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting bcd05fd7ab314f158b2cccbd3c3d02ac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731025557787 2024-11-08T00:25:59,285 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa290aea6aa343d88c1bd42f4932955e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731025557769 2024-11-08T00:25:59,292 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04acb16af9bbc0bb9b137adb2146a06b#A#compaction#469 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:25:59,292 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04acb16af9bbc0bb9b137adb2146a06b#B#compaction#470 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:25:59,293 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/fcf1885585b443a3af6e9d0b1b183f9c is 50, key is test_row_0/A:col10/1731025558439/Put/seqid=0 2024-11-08T00:25:59,293 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/f35771ee0ec94ed9b4248199ac3fec5d is 50, key is test_row_0/B:col10/1731025558439/Put/seqid=0 2024-11-08T00:25:59,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742381_1557 (size=12663) 2024-11-08T00:25:59,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742382_1558 (size=12663) 2024-11-08T00:25:59,324 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/fcf1885585b443a3af6e9d0b1b183f9c as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/fcf1885585b443a3af6e9d0b1b183f9c 2024-11-08T00:25:59,329 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 04acb16af9bbc0bb9b137adb2146a06b/A of 04acb16af9bbc0bb9b137adb2146a06b into fcf1885585b443a3af6e9d0b1b183f9c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:25:59,329 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04acb16af9bbc0bb9b137adb2146a06b: 2024-11-08T00:25:59,329 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., storeName=04acb16af9bbc0bb9b137adb2146a06b/A, priority=12, startTime=1731025559280; duration=0sec 2024-11-08T00:25:59,329 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:25:59,329 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04acb16af9bbc0bb9b137adb2146a06b:A 2024-11-08T00:25:59,329 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T00:25:59,330 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T00:25:59,331 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): 04acb16af9bbc0bb9b137adb2146a06b/C is initiating minor compaction (all files) 2024-11-08T00:25:59,331 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04acb16af9bbc0bb9b137adb2146a06b/C in TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:59,331 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/94d5cd355f3f49a1aee9ac9af6b1adae, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/abd1d452fc2146fca43432fa4d820585, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/f57a2f514e304a7db7624dabcd63c49a, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/8bbd663759b54b8ab21d59af15e1557b] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp, totalSize=47.8 K 2024-11-08T00:25:59,332 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94d5cd355f3f49a1aee9ac9af6b1adae, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731025555019 2024-11-08T00:25:59,332 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting abd1d452fc2146fca43432fa4d820585, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731025555115 2024-11-08T00:25:59,332 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting f57a2f514e304a7db7624dabcd63c49a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731025557055 2024-11-08T00:25:59,332 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bbd663759b54b8ab21d59af15e1557b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731025557787 2024-11-08T00:25:59,339 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04acb16af9bbc0bb9b137adb2146a06b#C#compaction#471 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:25:59,339 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/86149fec8657483f99c23f03cd869246 is 50, key is test_row_0/C:col10/1731025558439/Put/seqid=0 2024-11-08T00:25:59,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742383_1559 (size=12663) 2024-11-08T00:25:59,348 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/86149fec8657483f99c23f03cd869246 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/86149fec8657483f99c23f03cd869246 2024-11-08T00:25:59,352 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 04acb16af9bbc0bb9b137adb2146a06b/C of 04acb16af9bbc0bb9b137adb2146a06b into 86149fec8657483f99c23f03cd869246(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:25:59,352 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04acb16af9bbc0bb9b137adb2146a06b: 2024-11-08T00:25:59,352 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., storeName=04acb16af9bbc0bb9b137adb2146a06b/C, priority=12, startTime=1731025559280; duration=0sec 2024-11-08T00:25:59,352 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:25:59,353 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04acb16af9bbc0bb9b137adb2146a06b:C 2024-11-08T00:25:59,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 04acb16af9bbc0bb9b137adb2146a06b 2024-11-08T00:25:59,353 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:25:59,353 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 04acb16af9bbc0bb9b137adb2146a06b 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-08T00:25:59,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/deae58b4bbf24023865d006902bdbbe7 is 50, key is test_row_0/A:col10/1731025559352/Put/seqid=0 2024-11-08T00:25:59,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742384_1560 (size=16931) 2024-11-08T00:25:59,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:59,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50280 deadline: 1731025619407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:59,411 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:59,411 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:59,411 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:59,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:59,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50194 deadline: 1731025619408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:59,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:59,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:59,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50248 deadline: 1731025619410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:59,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:59,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:59,418 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:59,418 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:59,418 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:59,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-08T00:25:59,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:59,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. as already flushing 2024-11-08T00:25:59,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:59,426 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:59,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:59,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:59,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-08T00:25:59,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:59,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50280 deadline: 1731025619517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:59,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:59,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:59,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:59,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:59,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50194 deadline: 1731025619527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:59,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:59,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50248 deadline: 1731025619527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:59,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:59,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:59,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:59,531 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:59,531 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:59,531 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:59,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-08T00:25:59,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:59,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. as already flushing 2024-11-08T00:25:59,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:59,579 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:59,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:59,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:59,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:59,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50222 deadline: 1731025619667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:59,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:59,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:59,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:59,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:59,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1731025619667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:59,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:59,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:59,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:59,701 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/f35771ee0ec94ed9b4248199ac3fec5d as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/f35771ee0ec94ed9b4248199ac3fec5d 2024-11-08T00:25:59,705 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 04acb16af9bbc0bb9b137adb2146a06b/B of 04acb16af9bbc0bb9b137adb2146a06b into f35771ee0ec94ed9b4248199ac3fec5d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:25:59,705 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04acb16af9bbc0bb9b137adb2146a06b: 2024-11-08T00:25:59,705 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., storeName=04acb16af9bbc0bb9b137adb2146a06b/B, priority=12, startTime=1731025559280; duration=0sec 2024-11-08T00:25:59,705 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:25:59,705 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04acb16af9bbc0bb9b137adb2146a06b:B 2024-11-08T00:25:59,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:59,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50280 deadline: 1731025619727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:59,731 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:59,731 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:59,731 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:59,732 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-08T00:25:59,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:59,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. as already flushing 2024-11-08T00:25:59,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:59,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:59,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:59,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:59,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:59,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50194 deadline: 1731025619737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:59,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:59,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:59,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:59,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:25:59,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50248 deadline: 1731025619737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:25:59,743 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:59,743 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:25:59,743 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:25:59,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-08T00:25:59,779 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/deae58b4bbf24023865d006902bdbbe7 2024-11-08T00:25:59,784 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/0699517e6eea4ef5a988c20ff3f008c4 is 50, key is test_row_0/B:col10/1731025559352/Put/seqid=0 2024-11-08T00:25:59,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742385_1561 (size=12151) 2024-11-08T00:25:59,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-08T00:25:59,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:59,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. as already flushing 2024-11-08T00:25:59,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:25:59,885 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:59,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:25:59,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:00,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-08T00:26:00,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:26:00,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. as already flushing 2024-11-08T00:26:00,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:26:00,038 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:00,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:00,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:00,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:00,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50280 deadline: 1731025620037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:00,043 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:00,043 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:00,043 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:00,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:00,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50194 deadline: 1731025620047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:00,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:00,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:00,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:00,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:00,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50248 deadline: 1731025620048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:00,052 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:00,053 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:00,053 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:00,188 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/0699517e6eea4ef5a988c20ff3f008c4 2024-11-08T00:26:00,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-08T00:26:00,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:26:00,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. as already flushing 2024-11-08T00:26:00,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:26:00,190 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:00,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:00,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:00,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/14e8bba52b744775b1c6126393fe9853 is 50, key is test_row_0/C:col10/1731025559352/Put/seqid=0 2024-11-08T00:26:00,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742386_1562 (size=12151) 2024-11-08T00:26:00,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-08T00:26:00,343 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-08T00:26:00,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:26:00,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. as already flushing 2024-11-08T00:26:00,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:26:00,343 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:00,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:00,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:00,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-08T00:26:00,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:26:00,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. as already flushing 2024-11-08T00:26:00,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:26:00,496 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:00,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:00,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:00,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:00,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50280 deadline: 1731025620557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:00,563 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:00,563 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:00,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:00,563 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:00,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50194 deadline: 1731025620557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:00,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:00,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:00,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:00,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:00,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50248 deadline: 1731025620567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:00,572 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:00,572 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:00,572 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:00,598 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/14e8bba52b744775b1c6126393fe9853 2024-11-08T00:26:00,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/deae58b4bbf24023865d006902bdbbe7 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/deae58b4bbf24023865d006902bdbbe7 2024-11-08T00:26:00,604 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/deae58b4bbf24023865d006902bdbbe7, entries=250, sequenceid=234, filesize=16.5 K 2024-11-08T00:26:00,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/0699517e6eea4ef5a988c20ff3f008c4 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/0699517e6eea4ef5a988c20ff3f008c4 2024-11-08T00:26:00,608 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/0699517e6eea4ef5a988c20ff3f008c4, entries=150, sequenceid=234, filesize=11.9 K 2024-11-08T00:26:00,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/14e8bba52b744775b1c6126393fe9853 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/14e8bba52b744775b1c6126393fe9853 2024-11-08T00:26:00,612 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/14e8bba52b744775b1c6126393fe9853, entries=150, sequenceid=234, filesize=11.9 K 2024-11-08T00:26:00,612 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 04acb16af9bbc0bb9b137adb2146a06b in 1259ms, sequenceid=234, compaction requested=false 2024-11-08T00:26:00,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 04acb16af9bbc0bb9b137adb2146a06b: 2024-11-08T00:26:00,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-08T00:26:00,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:26:00,648 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 04acb16af9bbc0bb9b137adb2146a06b 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-08T00:26:00,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/e9115335fe2c4ca08327b82c30d5ee28 is 50, key is test_row_0/A:col10/1731025559409/Put/seqid=0 2024-11-08T00:26:00,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742387_1563 (size=12151) 2024-11-08T00:26:00,656 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/e9115335fe2c4ca08327b82c30d5ee28 2024-11-08T00:26:00,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/36ec2e9335c349a9877d4df72f856788 is 50, key is test_row_0/B:col10/1731025559409/Put/seqid=0 2024-11-08T00:26:00,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742388_1564 (size=12151) 2024-11-08T00:26:00,669 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/36ec2e9335c349a9877d4df72f856788 2024-11-08T00:26:00,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/9c3795acc0c14d56a33e4ceefe2805f9 is 50, key is test_row_0/C:col10/1731025559409/Put/seqid=0 2024-11-08T00:26:00,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742389_1565 (size=12151) 2024-11-08T00:26:00,684 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/9c3795acc0c14d56a33e4ceefe2805f9 2024-11-08T00:26:00,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/e9115335fe2c4ca08327b82c30d5ee28 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/e9115335fe2c4ca08327b82c30d5ee28 2024-11-08T00:26:00,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 04acb16af9bbc0bb9b137adb2146a06b 2024-11-08T00:26:00,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. as already flushing 2024-11-08T00:26:00,694 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/e9115335fe2c4ca08327b82c30d5ee28, entries=150, sequenceid=254, filesize=11.9 K 2024-11-08T00:26:00,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/36ec2e9335c349a9877d4df72f856788 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/36ec2e9335c349a9877d4df72f856788 2024-11-08T00:26:00,697 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/36ec2e9335c349a9877d4df72f856788, entries=150, sequenceid=254, filesize=11.9 K 2024-11-08T00:26:00,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/9c3795acc0c14d56a33e4ceefe2805f9 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/9c3795acc0c14d56a33e4ceefe2805f9 2024-11-08T00:26:00,703 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/9c3795acc0c14d56a33e4ceefe2805f9, entries=150, sequenceid=254, filesize=11.9 K 2024-11-08T00:26:00,704 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=26.84 KB/27480 for 04acb16af9bbc0bb9b137adb2146a06b in 56ms, sequenceid=254, compaction requested=true 2024-11-08T00:26:00,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 04acb16af9bbc0bb9b137adb2146a06b: 2024-11-08T00:26:00,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:26:00,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-08T00:26:00,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-08T00:26:00,709 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-08T00:26:00,709 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5870 sec 2024-11-08T00:26:00,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 04acb16af9bbc0bb9b137adb2146a06b 2024-11-08T00:26:00,710 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:26:00,710 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 04acb16af9bbc0bb9b137adb2146a06b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T00:26:00,711 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.5920 sec 2024-11-08T00:26:00,713 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/6a715ccc55d44efb833d8f03791c5ed5 is 50, key is test_row_0/A:col10/1731025560709/Put/seqid=0 2024-11-08T00:26:00,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742390_1566 (size=17181) 2024-11-08T00:26:00,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:00,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50222 deadline: 1731025620805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:00,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:00,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:00,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:00,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:00,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1731025620814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:00,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:00,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:00,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:00,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:00,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50222 deadline: 1731025620917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:00,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:00,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:00,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:00,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:00,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1731025620927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:00,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:00,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:00,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:01,117 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/6a715ccc55d44efb833d8f03791c5ed5 2024-11-08T00:26:01,122 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/f9f6b2c0bf2a4e46b6582ad2643920e7 is 50, key is test_row_0/B:col10/1731025560709/Put/seqid=0 2024-11-08T00:26:01,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742391_1567 (size=12301) 2024-11-08T00:26:01,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:01,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50222 deadline: 1731025621139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:01,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:01,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:01,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:01,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:01,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1731025621147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:01,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:01,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:01,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:01,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-08T00:26:01,267 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:26:01,268 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-08T00:26:01,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-08T00:26:01,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-08T00:26:01,270 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T00:26:01,270 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T00:26:01,270 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T00:26:01,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-08T00:26:01,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-08T00:26:01,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:26:01,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. as already flushing 2024-11-08T00:26:01,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:26:01,422 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:01,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:01,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:01,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:01,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50222 deadline: 1731025621457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:01,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:01,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:01,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:01,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:01,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1731025621467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:01,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:01,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:01,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:01,525 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/f9f6b2c0bf2a4e46b6582ad2643920e7 2024-11-08T00:26:01,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/c43005bfa65d41c7ba04c9084a97b12f is 50, key is test_row_0/C:col10/1731025560709/Put/seqid=0 2024-11-08T00:26:01,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742392_1568 (size=12301) 2024-11-08T00:26:01,536 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/c43005bfa65d41c7ba04c9084a97b12f 2024-11-08T00:26:01,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/6a715ccc55d44efb833d8f03791c5ed5 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/6a715ccc55d44efb833d8f03791c5ed5 2024-11-08T00:26:01,541 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/6a715ccc55d44efb833d8f03791c5ed5, entries=250, sequenceid=265, filesize=16.8 K 2024-11-08T00:26:01,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/f9f6b2c0bf2a4e46b6582ad2643920e7 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/f9f6b2c0bf2a4e46b6582ad2643920e7 2024-11-08T00:26:01,547 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/f9f6b2c0bf2a4e46b6582ad2643920e7, entries=150, sequenceid=265, filesize=12.0 K 2024-11-08T00:26:01,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/c43005bfa65d41c7ba04c9084a97b12f as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/c43005bfa65d41c7ba04c9084a97b12f 2024-11-08T00:26:01,551 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/c43005bfa65d41c7ba04c9084a97b12f, entries=150, sequenceid=265, filesize=12.0 K 2024-11-08T00:26:01,551 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 04acb16af9bbc0bb9b137adb2146a06b in 841ms, sequenceid=265, compaction requested=true 2024-11-08T00:26:01,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 04acb16af9bbc0bb9b137adb2146a06b: 2024-11-08T00:26:01,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04acb16af9bbc0bb9b137adb2146a06b:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T00:26:01,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:26:01,551 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T00:26:01,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04acb16af9bbc0bb9b137adb2146a06b:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T00:26:01,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:26:01,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04acb16af9bbc0bb9b137adb2146a06b:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:26:01,552 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:26:01,552 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T00:26:01,552 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 58926 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T00:26:01,552 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49266 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T00:26:01,552 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): 04acb16af9bbc0bb9b137adb2146a06b/A is initiating minor compaction (all files) 2024-11-08T00:26:01,552 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): 04acb16af9bbc0bb9b137adb2146a06b/B is initiating minor compaction (all files) 2024-11-08T00:26:01,552 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04acb16af9bbc0bb9b137adb2146a06b/A in TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:26:01,552 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04acb16af9bbc0bb9b137adb2146a06b/B in TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:26:01,552 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/fcf1885585b443a3af6e9d0b1b183f9c, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/deae58b4bbf24023865d006902bdbbe7, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/e9115335fe2c4ca08327b82c30d5ee28, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/6a715ccc55d44efb833d8f03791c5ed5] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp, totalSize=57.5 K 2024-11-08T00:26:01,552 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/f35771ee0ec94ed9b4248199ac3fec5d, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/0699517e6eea4ef5a988c20ff3f008c4, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/36ec2e9335c349a9877d4df72f856788, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/f9f6b2c0bf2a4e46b6582ad2643920e7] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp, totalSize=48.1 K 2024-11-08T00:26:01,553 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting fcf1885585b443a3af6e9d0b1b183f9c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731025557787 2024-11-08T00:26:01,553 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting f35771ee0ec94ed9b4248199ac3fec5d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731025557787 2024-11-08T00:26:01,553 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting deae58b4bbf24023865d006902bdbbe7, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731025558470 2024-11-08T00:26:01,553 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 0699517e6eea4ef5a988c20ff3f008c4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731025558509 2024-11-08T00:26:01,553 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9115335fe2c4ca08327b82c30d5ee28, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731025559397 2024-11-08T00:26:01,553 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 36ec2e9335c349a9877d4df72f856788, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731025559397 2024-11-08T00:26:01,553 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a715ccc55d44efb833d8f03791c5ed5, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731025560691 2024-11-08T00:26:01,554 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting f9f6b2c0bf2a4e46b6582ad2643920e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731025560705 2024-11-08T00:26:01,560 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04acb16af9bbc0bb9b137adb2146a06b#A#compaction#481 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:26:01,561 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/b016c0f48e0c4103adceb5ddb9dfbb9f is 50, key is test_row_0/A:col10/1731025560709/Put/seqid=0 2024-11-08T00:26:01,563 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04acb16af9bbc0bb9b137adb2146a06b#B#compaction#482 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:26:01,563 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/8ddb0e063ad4442fbaecb4a80d78216f is 50, key is test_row_0/B:col10/1731025560709/Put/seqid=0 2024-11-08T00:26:01,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742393_1569 (size=12949) 2024-11-08T00:26:01,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742394_1570 (size=12949) 2024-11-08T00:26:01,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 04acb16af9bbc0bb9b137adb2146a06b 2024-11-08T00:26:01,574 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:26:01,574 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 04acb16af9bbc0bb9b137adb2146a06b 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-08T00:26:01,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-08T00:26:01,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:26:01,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. as already flushing 2024-11-08T00:26:01,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:26:01,575 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:01,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:01,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:01,578 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/a282ce52cbdb4480b83dc973003986ea is 50, key is test_row_0/A:col10/1731025560812/Put/seqid=0 2024-11-08T00:26:01,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742395_1571 (size=14741) 2024-11-08T00:26:01,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-08T00:26:01,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:01,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50194 deadline: 1731025621603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:01,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:01,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:01,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:01,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50248 deadline: 1731025621604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:01,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:01,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:01,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:01,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:01,618 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:01,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50280 deadline: 1731025621606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:01,618 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:01,618 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:01,618 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:01,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-08T00:26:01,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:26:01,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. as already flushing 2024-11-08T00:26:01,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:26:01,728 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:01,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:01,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:01,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:01,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50194 deadline: 1731025621727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:01,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:01,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:01,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50248 deadline: 1731025621728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:01,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:01,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:01,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50280 deadline: 1731025621728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:01,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:01,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:01,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:01,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:01,738 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:01,738 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:01,738 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:01,880 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-08T00:26:01,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:26:01,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. as already flushing 2024-11-08T00:26:01,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:26:01,880 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:01,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:01,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:01,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-08T00:26:01,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:01,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50194 deadline: 1731025621948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:01,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:01,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:01,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50280 deadline: 1731025621948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:01,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:01,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:01,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:01,951 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:01,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50248 deadline: 1731025621948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:01,951 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:01,951 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:01,951 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:01,951 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:01,951 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:01,967 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/b016c0f48e0c4103adceb5ddb9dfbb9f as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/b016c0f48e0c4103adceb5ddb9dfbb9f 2024-11-08T00:26:01,973 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/8ddb0e063ad4442fbaecb4a80d78216f as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/8ddb0e063ad4442fbaecb4a80d78216f 2024-11-08T00:26:01,974 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 04acb16af9bbc0bb9b137adb2146a06b/A of 04acb16af9bbc0bb9b137adb2146a06b into b016c0f48e0c4103adceb5ddb9dfbb9f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:26:01,974 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04acb16af9bbc0bb9b137adb2146a06b: 2024-11-08T00:26:01,974 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., storeName=04acb16af9bbc0bb9b137adb2146a06b/A, priority=12, startTime=1731025561551; duration=0sec 2024-11-08T00:26:01,974 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:26:01,974 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04acb16af9bbc0bb9b137adb2146a06b:A 2024-11-08T00:26:01,975 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T00:26:01,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:01,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50222 deadline: 1731025621967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:01,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:01,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:01,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:01,976 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49266 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T00:26:01,976 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): 04acb16af9bbc0bb9b137adb2146a06b/C is initiating minor compaction (all files) 2024-11-08T00:26:01,977 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04acb16af9bbc0bb9b137adb2146a06b/C in TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:26:01,977 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/86149fec8657483f99c23f03cd869246, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/14e8bba52b744775b1c6126393fe9853, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/9c3795acc0c14d56a33e4ceefe2805f9, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/c43005bfa65d41c7ba04c9084a97b12f] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp, totalSize=48.1 K 2024-11-08T00:26:01,977 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86149fec8657483f99c23f03cd869246, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731025557787 2024-11-08T00:26:01,977 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14e8bba52b744775b1c6126393fe9853, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731025558509 2024-11-08T00:26:01,978 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c3795acc0c14d56a33e4ceefe2805f9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731025559397 2024-11-08T00:26:01,978 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting c43005bfa65d41c7ba04c9084a97b12f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731025560705 2024-11-08T00:26:01,979 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 04acb16af9bbc0bb9b137adb2146a06b/B of 04acb16af9bbc0bb9b137adb2146a06b into 8ddb0e063ad4442fbaecb4a80d78216f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:26:01,979 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04acb16af9bbc0bb9b137adb2146a06b: 2024-11-08T00:26:01,979 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., storeName=04acb16af9bbc0bb9b137adb2146a06b/B, priority=12, startTime=1731025561551; duration=0sec 2024-11-08T00:26:01,979 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:26:01,979 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04acb16af9bbc0bb9b137adb2146a06b:B 2024-11-08T00:26:01,981 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/a282ce52cbdb4480b83dc973003986ea 2024-11-08T00:26:01,984 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04acb16af9bbc0bb9b137adb2146a06b#C#compaction#484 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:26:01,984 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/c564f830945848dd8951c0c12cdc106b is 50, key is test_row_0/C:col10/1731025560709/Put/seqid=0 2024-11-08T00:26:01,987 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/665e99802f3d49018f0f47c70404f8da is 50, key is test_row_0/B:col10/1731025560812/Put/seqid=0 2024-11-08T00:26:01,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:01,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1731025621977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:01,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:01,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:01,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:01,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742396_1572 (size=12949) 2024-11-08T00:26:02,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742397_1573 (size=12301) 2024-11-08T00:26:02,006 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/665e99802f3d49018f0f47c70404f8da 2024-11-08T00:26:02,016 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/26f16580c114477ca2585df28a745895 is 50, key is test_row_0/C:col10/1731025560812/Put/seqid=0 2024-11-08T00:26:02,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742398_1574 (size=12301) 2024-11-08T00:26:02,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-08T00:26:02,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:26:02,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. as already flushing 2024-11-08T00:26:02,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:26:02,033 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:02,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:02,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:02,187 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-08T00:26:02,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:26:02,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. as already flushing 2024-11-08T00:26:02,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:26:02,187 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:02,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:02,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:02,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:02,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50194 deadline: 1731025622257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:02,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:02,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:02,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:02,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50248 deadline: 1731025622257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:02,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:02,264 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:02,264 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:02,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:02,264 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:02,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50280 deadline: 1731025622258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:02,265 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:02,265 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:02,265 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:02,340 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-08T00:26:02,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:26:02,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. as already flushing 2024-11-08T00:26:02,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:26:02,340 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:02,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:02,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:02,397 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/c564f830945848dd8951c0c12cdc106b as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/c564f830945848dd8951c0c12cdc106b 2024-11-08T00:26:02,401 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 04acb16af9bbc0bb9b137adb2146a06b/C of 04acb16af9bbc0bb9b137adb2146a06b into c564f830945848dd8951c0c12cdc106b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:26:02,401 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04acb16af9bbc0bb9b137adb2146a06b: 2024-11-08T00:26:02,401 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., storeName=04acb16af9bbc0bb9b137adb2146a06b/C, priority=12, startTime=1731025561551; duration=0sec 2024-11-08T00:26:02,401 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:26:02,401 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04acb16af9bbc0bb9b137adb2146a06b:C 2024-11-08T00:26:02,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-08T00:26:02,420 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/26f16580c114477ca2585df28a745895 2024-11-08T00:26:02,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/a282ce52cbdb4480b83dc973003986ea as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/a282ce52cbdb4480b83dc973003986ea 2024-11-08T00:26:02,427 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/a282ce52cbdb4480b83dc973003986ea, entries=200, sequenceid=291, filesize=14.4 K 2024-11-08T00:26:02,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/665e99802f3d49018f0f47c70404f8da as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/665e99802f3d49018f0f47c70404f8da 2024-11-08T00:26:02,429 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/665e99802f3d49018f0f47c70404f8da, entries=150, sequenceid=291, filesize=12.0 K 2024-11-08T00:26:02,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/26f16580c114477ca2585df28a745895 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/26f16580c114477ca2585df28a745895 2024-11-08T00:26:02,434 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/26f16580c114477ca2585df28a745895, entries=150, sequenceid=291, filesize=12.0 K 2024-11-08T00:26:02,435 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 04acb16af9bbc0bb9b137adb2146a06b in 860ms, sequenceid=291, compaction requested=false 2024-11-08T00:26:02,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 04acb16af9bbc0bb9b137adb2146a06b: 2024-11-08T00:26:02,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-08T00:26:02,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:26:02,493 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 04acb16af9bbc0bb9b137adb2146a06b 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-08T00:26:02,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/60ccf8011ce94a379edb2756f0c13df9 is 50, key is test_row_0/A:col10/1731025561598/Put/seqid=0 2024-11-08T00:26:02,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742399_1575 (size=12301) 2024-11-08T00:26:02,501 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/60ccf8011ce94a379edb2756f0c13df9 2024-11-08T00:26:02,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/1d37d0f29c7f4ad88fbb676f2fc50d47 is 50, key is test_row_0/B:col10/1731025561598/Put/seqid=0 2024-11-08T00:26:02,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742400_1576 (size=12301) 2024-11-08T00:26:02,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 04acb16af9bbc0bb9b137adb2146a06b 2024-11-08T00:26:02,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. as already flushing 2024-11-08T00:26:02,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:02,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50194 deadline: 1731025622889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:02,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:02,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:02,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:02,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:02,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50280 deadline: 1731025622890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:02,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:02,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:02,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:02,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:02,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50248 deadline: 1731025622896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:02,904 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:02,904 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:02,904 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:02,910 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/1d37d0f29c7f4ad88fbb676f2fc50d47 2024-11-08T00:26:02,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/bc81c62175fc4449a19b5378cedf3a4c is 50, key is test_row_0/C:col10/1731025561598/Put/seqid=0 2024-11-08T00:26:02,918 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T00:26:02,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742401_1577 (size=12301) 2024-11-08T00:26:02,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:02,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50222 deadline: 1731025622987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:02,996 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 3302f0f507bd,40479,1731025415856 for 'test_row_1' in 04acb16af9bbc0bb9b137adb2146a06b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T00:26:02,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:02,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:02,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:03,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:03,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1731025622997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:03,006 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 3302f0f507bd,40479,1731025415856 for 'test_row_0' in 04acb16af9bbc0bb9b137adb2146a06b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-08T00:26:03,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:03,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:03,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:03,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:03,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50194 deadline: 1731025623007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:03,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:03,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:03,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:03,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:03,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50280 deadline: 1731025623007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:03,016 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:03,016 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:03,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:03,016 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:03,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50248 deadline: 1731025623008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:03,016 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:03,016 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:03,016 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:03,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:03,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50194 deadline: 1731025623227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:03,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:03,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:03,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:03,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:03,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50280 deadline: 1731025623227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:03,229 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:03,229 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:03,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:03,229 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:03,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50248 deadline: 1731025623227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:03,230 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:03,230 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:03,230 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:03,319 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/bc81c62175fc4449a19b5378cedf3a4c 2024-11-08T00:26:03,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/60ccf8011ce94a379edb2756f0c13df9 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/60ccf8011ce94a379edb2756f0c13df9 2024-11-08T00:26:03,325 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/60ccf8011ce94a379edb2756f0c13df9, entries=150, sequenceid=304, filesize=12.0 K 2024-11-08T00:26:03,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/1d37d0f29c7f4ad88fbb676f2fc50d47 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/1d37d0f29c7f4ad88fbb676f2fc50d47 2024-11-08T00:26:03,328 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/1d37d0f29c7f4ad88fbb676f2fc50d47, entries=150, sequenceid=304, filesize=12.0 K 2024-11-08T00:26:03,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/bc81c62175fc4449a19b5378cedf3a4c as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/bc81c62175fc4449a19b5378cedf3a4c 2024-11-08T00:26:03,331 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/bc81c62175fc4449a19b5378cedf3a4c, entries=150, sequenceid=304, filesize=12.0 K 2024-11-08T00:26:03,332 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 04acb16af9bbc0bb9b137adb2146a06b in 839ms, sequenceid=304, compaction requested=true 2024-11-08T00:26:03,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 04acb16af9bbc0bb9b137adb2146a06b: 2024-11-08T00:26:03,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:26:03,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-08T00:26:03,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-08T00:26:03,335 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-08T00:26:03,335 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0630 sec 2024-11-08T00:26:03,337 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 2.0680 sec 2024-11-08T00:26:03,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-08T00:26:03,427 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:26:03,427 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-08T00:26:03,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-08T00:26:03,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-08T00:26:03,429 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T00:26:03,430 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T00:26:03,430 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T00:26:03,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-08T00:26:03,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 04acb16af9bbc0bb9b137adb2146a06b 2024-11-08T00:26:03,542 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:26:03,542 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 04acb16af9bbc0bb9b137adb2146a06b 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-08T00:26:03,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/2027c35d198f4525b52f84113c505b55 is 50, key is test_row_0/A:col10/1731025562881/Put/seqid=0 2024-11-08T00:26:03,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742402_1578 (size=17181) 2024-11-08T00:26:03,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:03,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50280 deadline: 1731025623556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:03,559 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:03,560 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:03,560 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:03,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:03,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50248 deadline: 1731025623558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:03,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:03,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50194 deadline: 1731025623559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:03,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:03,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:03,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:03,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:03,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:03,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:03,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-08T00:26:03,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:26:03,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. as already flushing 2024-11-08T00:26:03,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:26:03,582 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:03,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:03,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:03,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:03,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50280 deadline: 1731025623667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:03,673 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:03,673 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:03,673 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:03,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:03,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50248 deadline: 1731025623678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:03,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:03,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:03,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:03,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:03,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50194 deadline: 1731025623678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:03,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:03,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:03,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:03,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-08T00:26:03,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:26:03,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. as already flushing 2024-11-08T00:26:03,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:26:03,734 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:03,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:03,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:03,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-08T00:26:03,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:03,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50280 deadline: 1731025623877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:03,884 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:03,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:03,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:03,886 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-08T00:26:03,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:26:03,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. as already flushing 2024-11-08T00:26:03,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:26:03,887 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:03,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:03,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:03,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:03,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50248 deadline: 1731025623887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:03,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:03,898 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:03,898 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:03,898 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:03,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50194 deadline: 1731025623888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:03,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:03,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:03,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:03,950 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/2027c35d198f4525b52f84113c505b55 2024-11-08T00:26:03,956 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/491a1a9402334a658cbfa0443dcd0a6d is 50, key is test_row_0/B:col10/1731025562881/Put/seqid=0 2024-11-08T00:26:03,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742403_1579 (size=12301) 2024-11-08T00:26:03,964 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/491a1a9402334a658cbfa0443dcd0a6d 2024-11-08T00:26:03,970 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/8705030c5f5c4c7ea3e73344af383238 is 50, key is test_row_0/C:col10/1731025562881/Put/seqid=0 2024-11-08T00:26:03,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742404_1580 (size=12301) 2024-11-08T00:26:04,039 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-08T00:26:04,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:26:04,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. as already flushing 2024-11-08T00:26:04,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:26:04,039 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:04,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:04,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:04,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-08T00:26:04,191 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-08T00:26:04,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:04,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:26:04,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50280 deadline: 1731025624187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:04,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. as already flushing 2024-11-08T00:26:04,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:26:04,191 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:04,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:04,192 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:04,192 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:04,192 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:04,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:04,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:04,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50248 deadline: 1731025624207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:04,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:04,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:04,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:04,214 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:04,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50194 deadline: 1731025624207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:04,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:04,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:04,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:04,344 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-08T00:26:04,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:26:04,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. as already flushing 2024-11-08T00:26:04,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:26:04,344 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:04,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:04,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:04,378 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/8705030c5f5c4c7ea3e73344af383238 2024-11-08T00:26:04,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/2027c35d198f4525b52f84113c505b55 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/2027c35d198f4525b52f84113c505b55 2024-11-08T00:26:04,383 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/2027c35d198f4525b52f84113c505b55, entries=250, sequenceid=332, filesize=16.8 K 2024-11-08T00:26:04,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/491a1a9402334a658cbfa0443dcd0a6d as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/491a1a9402334a658cbfa0443dcd0a6d 2024-11-08T00:26:04,388 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/491a1a9402334a658cbfa0443dcd0a6d, entries=150, sequenceid=332, filesize=12.0 K 2024-11-08T00:26:04,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/8705030c5f5c4c7ea3e73344af383238 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/8705030c5f5c4c7ea3e73344af383238 2024-11-08T00:26:04,391 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/8705030c5f5c4c7ea3e73344af383238, entries=150, sequenceid=332, filesize=12.0 K 2024-11-08T00:26:04,392 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 04acb16af9bbc0bb9b137adb2146a06b in 850ms, sequenceid=332, compaction requested=true 2024-11-08T00:26:04,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 04acb16af9bbc0bb9b137adb2146a06b: 2024-11-08T00:26:04,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04acb16af9bbc0bb9b137adb2146a06b:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T00:26:04,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:26:04,392 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T00:26:04,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04acb16af9bbc0bb9b137adb2146a06b:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T00:26:04,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:26:04,392 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T00:26:04,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04acb16af9bbc0bb9b137adb2146a06b:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:26:04,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:26:04,393 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57172 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T00:26:04,393 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T00:26:04,394 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): 04acb16af9bbc0bb9b137adb2146a06b/A is initiating minor compaction (all files) 2024-11-08T00:26:04,394 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): 04acb16af9bbc0bb9b137adb2146a06b/B is initiating minor compaction (all files) 2024-11-08T00:26:04,394 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04acb16af9bbc0bb9b137adb2146a06b/B in TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:26:04,394 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04acb16af9bbc0bb9b137adb2146a06b/A in TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:26:04,394 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/8ddb0e063ad4442fbaecb4a80d78216f, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/665e99802f3d49018f0f47c70404f8da, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/1d37d0f29c7f4ad88fbb676f2fc50d47, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/491a1a9402334a658cbfa0443dcd0a6d] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp, totalSize=48.7 K 2024-11-08T00:26:04,394 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/b016c0f48e0c4103adceb5ddb9dfbb9f, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/a282ce52cbdb4480b83dc973003986ea, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/60ccf8011ce94a379edb2756f0c13df9, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/2027c35d198f4525b52f84113c505b55] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp, totalSize=55.8 K 2024-11-08T00:26:04,394 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ddb0e063ad4442fbaecb4a80d78216f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731025560705 2024-11-08T00:26:04,394 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting b016c0f48e0c4103adceb5ddb9dfbb9f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731025560705 2024-11-08T00:26:04,394 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 665e99802f3d49018f0f47c70404f8da, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731025560784 2024-11-08T00:26:04,394 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting a282ce52cbdb4480b83dc973003986ea, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731025560784 2024-11-08T00:26:04,394 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d37d0f29c7f4ad88fbb676f2fc50d47, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731025561598 2024-11-08T00:26:04,394 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60ccf8011ce94a379edb2756f0c13df9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731025561598 2024-11-08T00:26:04,395 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 491a1a9402334a658cbfa0443dcd0a6d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731025562881 2024-11-08T00:26:04,395 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2027c35d198f4525b52f84113c505b55, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731025562881 2024-11-08T00:26:04,403 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04acb16af9bbc0bb9b137adb2146a06b#B#compaction#493 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:26:04,403 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04acb16af9bbc0bb9b137adb2146a06b#A#compaction#494 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:26:04,404 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/fda93b2578824b34b0e458553d5616a0 is 50, key is test_row_0/B:col10/1731025562881/Put/seqid=0 2024-11-08T00:26:04,404 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/1c38c3a4e2b54053af7dee1733a85c00 is 50, key is test_row_0/A:col10/1731025562881/Put/seqid=0 2024-11-08T00:26:04,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742405_1581 (size=13085) 2024-11-08T00:26:04,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742406_1582 (size=13085) 2024-11-08T00:26:04,414 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/1c38c3a4e2b54053af7dee1733a85c00 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/1c38c3a4e2b54053af7dee1733a85c00 2024-11-08T00:26:04,418 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 04acb16af9bbc0bb9b137adb2146a06b/A of 04acb16af9bbc0bb9b137adb2146a06b into 1c38c3a4e2b54053af7dee1733a85c00(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:26:04,418 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04acb16af9bbc0bb9b137adb2146a06b: 2024-11-08T00:26:04,418 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., storeName=04acb16af9bbc0bb9b137adb2146a06b/A, priority=12, startTime=1731025564392; duration=0sec 2024-11-08T00:26:04,418 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:26:04,418 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04acb16af9bbc0bb9b137adb2146a06b:A 2024-11-08T00:26:04,418 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T00:26:04,419 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T00:26:04,419 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): 04acb16af9bbc0bb9b137adb2146a06b/C is initiating minor compaction (all files) 2024-11-08T00:26:04,419 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04acb16af9bbc0bb9b137adb2146a06b/C in TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:26:04,420 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/c564f830945848dd8951c0c12cdc106b, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/26f16580c114477ca2585df28a745895, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/bc81c62175fc4449a19b5378cedf3a4c, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/8705030c5f5c4c7ea3e73344af383238] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp, totalSize=48.7 K 2024-11-08T00:26:04,420 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting c564f830945848dd8951c0c12cdc106b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731025560705 2024-11-08T00:26:04,420 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26f16580c114477ca2585df28a745895, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731025560784 2024-11-08T00:26:04,420 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc81c62175fc4449a19b5378cedf3a4c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731025561598 2024-11-08T00:26:04,420 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8705030c5f5c4c7ea3e73344af383238, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731025562881 2024-11-08T00:26:04,436 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04acb16af9bbc0bb9b137adb2146a06b#C#compaction#495 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:26:04,437 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/2db28c87fbcf4b06a139c2a530ec2985 is 50, key is test_row_0/C:col10/1731025562881/Put/seqid=0 2024-11-08T00:26:04,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742407_1583 (size=13085) 2024-11-08T00:26:04,450 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/2db28c87fbcf4b06a139c2a530ec2985 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/2db28c87fbcf4b06a139c2a530ec2985 2024-11-08T00:26:04,453 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 04acb16af9bbc0bb9b137adb2146a06b/C of 04acb16af9bbc0bb9b137adb2146a06b into 2db28c87fbcf4b06a139c2a530ec2985(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:26:04,453 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04acb16af9bbc0bb9b137adb2146a06b: 2024-11-08T00:26:04,453 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., storeName=04acb16af9bbc0bb9b137adb2146a06b/C, priority=12, startTime=1731025564392; duration=0sec 2024-11-08T00:26:04,453 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:26:04,453 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04acb16af9bbc0bb9b137adb2146a06b:C 2024-11-08T00:26:04,496 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-08T00:26:04,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:26:04,496 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 04acb16af9bbc0bb9b137adb2146a06b 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-08T00:26:04,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/123af6a9160f49daaf635a88942c260c is 50, key is test_row_0/A:col10/1731025563555/Put/seqid=0 2024-11-08T00:26:04,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742408_1584 (size=12301) 2024-11-08T00:26:04,515 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/123af6a9160f49daaf635a88942c260c 2024-11-08T00:26:04,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/a2159f1ac90f41b18482ffa2684ebf3c is 50, key is test_row_0/B:col10/1731025563555/Put/seqid=0 2024-11-08T00:26:04,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742409_1585 (size=12301) 2024-11-08T00:26:04,527 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/a2159f1ac90f41b18482ffa2684ebf3c 2024-11-08T00:26:04,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/cab46df99a84441da6cd2c21c781e2cf is 50, key is test_row_0/C:col10/1731025563555/Put/seqid=0 2024-11-08T00:26:04,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742410_1586 (size=12301) 2024-11-08T00:26:04,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-08T00:26:04,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 04acb16af9bbc0bb9b137adb2146a06b 2024-11-08T00:26:04,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. as already flushing 2024-11-08T00:26:04,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:04,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50248 deadline: 1731025624775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:04,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:04,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:04,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:04,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:04,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50280 deadline: 1731025624776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:04,783 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:04,783 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:04,783 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:04,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:04,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50194 deadline: 1731025624782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:04,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:04,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:04,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:04,813 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/fda93b2578824b34b0e458553d5616a0 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/fda93b2578824b34b0e458553d5616a0 2024-11-08T00:26:04,816 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 04acb16af9bbc0bb9b137adb2146a06b/B of 04acb16af9bbc0bb9b137adb2146a06b into fda93b2578824b34b0e458553d5616a0(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:26:04,816 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04acb16af9bbc0bb9b137adb2146a06b: 2024-11-08T00:26:04,816 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., storeName=04acb16af9bbc0bb9b137adb2146a06b/B, priority=12, startTime=1731025564392; duration=0sec 2024-11-08T00:26:04,816 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:26:04,816 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04acb16af9bbc0bb9b137adb2146a06b:B 2024-11-08T00:26:04,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:04,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50280 deadline: 1731025624887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:04,887 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:04,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:04,887 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:04,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:04,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50194 deadline: 1731025624887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:04,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:04,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:04,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:04,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:04,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50248 deadline: 1731025624887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:04,888 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:04,888 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:04,888 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:04,939 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/cab46df99a84441da6cd2c21c781e2cf 2024-11-08T00:26:04,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/123af6a9160f49daaf635a88942c260c as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/123af6a9160f49daaf635a88942c260c 2024-11-08T00:26:04,945 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/123af6a9160f49daaf635a88942c260c, entries=150, sequenceid=342, filesize=12.0 K 2024-11-08T00:26:04,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/a2159f1ac90f41b18482ffa2684ebf3c as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/a2159f1ac90f41b18482ffa2684ebf3c 2024-11-08T00:26:04,948 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/a2159f1ac90f41b18482ffa2684ebf3c, entries=150, sequenceid=342, filesize=12.0 K 2024-11-08T00:26:04,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/cab46df99a84441da6cd2c21c781e2cf as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/cab46df99a84441da6cd2c21c781e2cf 2024-11-08T00:26:04,953 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/cab46df99a84441da6cd2c21c781e2cf, entries=150, sequenceid=342, filesize=12.0 K 2024-11-08T00:26:04,954 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=167.72 KB/171750 for 04acb16af9bbc0bb9b137adb2146a06b in 457ms, sequenceid=342, compaction requested=false 2024-11-08T00:26:04,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 04acb16af9bbc0bb9b137adb2146a06b: 2024-11-08T00:26:04,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:26:04,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-08T00:26:04,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-08T00:26:04,956 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-08T00:26:04,956 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5250 sec 2024-11-08T00:26:04,957 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.5280 sec 2024-11-08T00:26:05,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 04acb16af9bbc0bb9b137adb2146a06b 2024-11-08T00:26:05,022 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:26:05,022 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 04acb16af9bbc0bb9b137adb2146a06b 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-08T00:26:05,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/2696f838495a4be9965772f25a82966f is 50, key is test_row_0/A:col10/1731025564775/Put/seqid=0 2024-11-08T00:26:05,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742411_1587 (size=14741) 2024-11-08T00:26:05,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:05,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1731025625040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:05,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:05,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:05,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:05,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:05,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50222 deadline: 1731025625040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:05,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:05,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:05,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:05,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:05,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50280 deadline: 1731025625097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:05,099 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:05,099 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:05,099 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:05,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:05,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50194 deadline: 1731025625097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:05,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:05,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:05,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:05,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:05,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50248 deadline: 1731025625097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:05,100 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:05,100 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:05,100 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:05,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:05,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1731025625157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:05,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:05,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:05,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:05,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:05,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50222 deadline: 1731025625157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:05,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:05,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:05,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:05,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:05,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:05,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50222 deadline: 1731025625367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:05,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1731025625367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:05,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:05,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:05,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:05,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:05,373 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:05,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:05,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:05,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50280 deadline: 1731025625407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:05,413 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:05,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:05,413 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:05,413 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:05,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50194 deadline: 1731025625407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:05,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:05,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:05,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:05,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:05,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50248 deadline: 1731025625407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:05,413 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:05,413 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:05,413 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:05,429 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/2696f838495a4be9965772f25a82966f 2024-11-08T00:26:05,435 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/400e103bd272405aab6d358cb0450b68 is 50, key is test_row_0/B:col10/1731025564775/Put/seqid=0 2024-11-08T00:26:05,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742412_1588 (size=12301) 2024-11-08T00:26:05,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-08T00:26:05,577 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:26:05,577 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-08T00:26:05,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-08T00:26:05,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-08T00:26:05,579 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T00:26:05,580 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T00:26:05,580 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T00:26:05,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:05,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1731025625677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:05,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:05,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:05,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50222 deadline: 1731025625677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:05,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:05,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:05,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:05,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:05,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:05,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-08T00:26:05,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-08T00:26:05,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:26:05,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. as already flushing 2024-11-08T00:26:05,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:26:05,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:05,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:05,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:05,838 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/400e103bd272405aab6d358cb0450b68 2024-11-08T00:26:05,843 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/a3f9f277852a4ece9e67eda74262f815 is 50, key is test_row_0/C:col10/1731025564775/Put/seqid=0 2024-11-08T00:26:05,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742413_1589 (size=12301) 2024-11-08T00:26:05,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-08T00:26:05,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:26:05,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. as already flushing 2024-11-08T00:26:05,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:26:05,885 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:05,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:05,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:05,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-08T00:26:05,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:05,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50280 deadline: 1731025625917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:05,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:05,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:05,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50248 deadline: 1731025625918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:05,921 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:05,921 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:05,921 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:05,921 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:05,921 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:05,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:05,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50194 deadline: 1731025625927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:05,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:05,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:05,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:06,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-08T00:26:06,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:26:06,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. as already flushing 2024-11-08T00:26:06,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:26:06,037 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:06,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:06,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:06,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-08T00:26:06,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:26:06,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. as already flushing 2024-11-08T00:26:06,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:26:06,190 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:06,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:06,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:06,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:06,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50222 deadline: 1731025626187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:06,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:06,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1731025626187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:06,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:06,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:06,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:06,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 , the old value is region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:06,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:06,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., hostname=3302f0f507bd,40479,1731025415856, seqNum=2 because the exception is null or not the one we care about 2024-11-08T00:26:06,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-08T00:26:06,252 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/a3f9f277852a4ece9e67eda74262f815 2024-11-08T00:26:06,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/2696f838495a4be9965772f25a82966f as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/2696f838495a4be9965772f25a82966f 2024-11-08T00:26:06,259 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/2696f838495a4be9965772f25a82966f, entries=200, sequenceid=372, filesize=14.4 K 2024-11-08T00:26:06,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/400e103bd272405aab6d358cb0450b68 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/400e103bd272405aab6d358cb0450b68 2024-11-08T00:26:06,264 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/400e103bd272405aab6d358cb0450b68, entries=150, sequenceid=372, filesize=12.0 K 2024-11-08T00:26:06,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/a3f9f277852a4ece9e67eda74262f815 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/a3f9f277852a4ece9e67eda74262f815 2024-11-08T00:26:06,267 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/a3f9f277852a4ece9e67eda74262f815, entries=150, sequenceid=372, filesize=12.0 K 2024-11-08T00:26:06,268 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for 04acb16af9bbc0bb9b137adb2146a06b in 1246ms, sequenceid=372, compaction requested=true 2024-11-08T00:26:06,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 04acb16af9bbc0bb9b137adb2146a06b: 2024-11-08T00:26:06,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04acb16af9bbc0bb9b137adb2146a06b:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T00:26:06,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:26:06,268 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:26:06,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04acb16af9bbc0bb9b137adb2146a06b:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T00:26:06,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:26:06,268 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:26:06,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04acb16af9bbc0bb9b137adb2146a06b:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:26:06,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:26:06,269 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40127 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:26:06,269 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:26:06,269 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): 04acb16af9bbc0bb9b137adb2146a06b/A is initiating minor compaction (all files) 2024-11-08T00:26:06,269 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): 04acb16af9bbc0bb9b137adb2146a06b/B is initiating minor compaction (all files) 2024-11-08T00:26:06,269 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04acb16af9bbc0bb9b137adb2146a06b/B in TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:26:06,269 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04acb16af9bbc0bb9b137adb2146a06b/A in TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:26:06,269 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/fda93b2578824b34b0e458553d5616a0, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/a2159f1ac90f41b18482ffa2684ebf3c, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/400e103bd272405aab6d358cb0450b68] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp, totalSize=36.8 K 2024-11-08T00:26:06,269 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/1c38c3a4e2b54053af7dee1733a85c00, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/123af6a9160f49daaf635a88942c260c, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/2696f838495a4be9965772f25a82966f] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp, totalSize=39.2 K 2024-11-08T00:26:06,269 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting fda93b2578824b34b0e458553d5616a0, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731025562881 2024-11-08T00:26:06,269 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c38c3a4e2b54053af7dee1733a85c00, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731025562881 2024-11-08T00:26:06,269 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting a2159f1ac90f41b18482ffa2684ebf3c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731025563547 2024-11-08T00:26:06,270 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 123af6a9160f49daaf635a88942c260c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731025563547 2024-11-08T00:26:06,270 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 400e103bd272405aab6d358cb0450b68, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731025564775 2024-11-08T00:26:06,270 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2696f838495a4be9965772f25a82966f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731025564742 2024-11-08T00:26:06,275 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04acb16af9bbc0bb9b137adb2146a06b#A#compaction#502 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:26:06,275 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/394b4767d94a48fe951ae57a050df01f is 50, key is test_row_0/A:col10/1731025564775/Put/seqid=0 2024-11-08T00:26:06,278 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04acb16af9bbc0bb9b137adb2146a06b#B#compaction#503 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:26:06,278 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/1791701f8a8c4219a0f6f8cd569082df is 50, key is test_row_0/B:col10/1731025564775/Put/seqid=0 2024-11-08T00:26:06,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742414_1590 (size=13187) 2024-11-08T00:26:06,285 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/394b4767d94a48fe951ae57a050df01f as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/394b4767d94a48fe951ae57a050df01f 2024-11-08T00:26:06,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742415_1591 (size=13187) 2024-11-08T00:26:06,293 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 04acb16af9bbc0bb9b137adb2146a06b/A of 04acb16af9bbc0bb9b137adb2146a06b into 394b4767d94a48fe951ae57a050df01f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:26:06,293 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04acb16af9bbc0bb9b137adb2146a06b: 2024-11-08T00:26:06,293 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., storeName=04acb16af9bbc0bb9b137adb2146a06b/A, priority=13, startTime=1731025566268; duration=0sec 2024-11-08T00:26:06,293 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:26:06,293 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04acb16af9bbc0bb9b137adb2146a06b:A 2024-11-08T00:26:06,293 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:26:06,293 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/1791701f8a8c4219a0f6f8cd569082df as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/1791701f8a8c4219a0f6f8cd569082df 2024-11-08T00:26:06,301 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:26:06,301 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): 04acb16af9bbc0bb9b137adb2146a06b/C is initiating minor compaction (all files) 2024-11-08T00:26:06,301 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04acb16af9bbc0bb9b137adb2146a06b/C in TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:26:06,301 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/2db28c87fbcf4b06a139c2a530ec2985, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/cab46df99a84441da6cd2c21c781e2cf, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/a3f9f277852a4ece9e67eda74262f815] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp, totalSize=36.8 K 2024-11-08T00:26:06,302 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2db28c87fbcf4b06a139c2a530ec2985, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731025562881 2024-11-08T00:26:06,320 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 04acb16af9bbc0bb9b137adb2146a06b/B of 04acb16af9bbc0bb9b137adb2146a06b into 1791701f8a8c4219a0f6f8cd569082df(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:26:06,320 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04acb16af9bbc0bb9b137adb2146a06b: 2024-11-08T00:26:06,320 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., storeName=04acb16af9bbc0bb9b137adb2146a06b/B, priority=13, startTime=1731025566268; duration=0sec 2024-11-08T00:26:06,320 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:26:06,320 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04acb16af9bbc0bb9b137adb2146a06b:B 2024-11-08T00:26:06,321 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting cab46df99a84441da6cd2c21c781e2cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731025563547 2024-11-08T00:26:06,325 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3f9f277852a4ece9e67eda74262f815, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731025564775 2024-11-08T00:26:06,334 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04acb16af9bbc0bb9b137adb2146a06b#C#compaction#504 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:26:06,334 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/eb0a388c46544962a7a3b34f1f7af161 is 50, key is test_row_0/C:col10/1731025564775/Put/seqid=0 2024-11-08T00:26:06,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742416_1592 (size=13187) 2024-11-08T00:26:06,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-08T00:26:06,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:26:06,342 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing 04acb16af9bbc0bb9b137adb2146a06b 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-08T00:26:06,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/cb040097fd284283ad3dc128ccc6541f is 50, key is test_row_0/A:col10/1731025565031/Put/seqid=0 2024-11-08T00:26:06,348 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/eb0a388c46544962a7a3b34f1f7af161 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/eb0a388c46544962a7a3b34f1f7af161 2024-11-08T00:26:06,352 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 04acb16af9bbc0bb9b137adb2146a06b/C of 04acb16af9bbc0bb9b137adb2146a06b into eb0a388c46544962a7a3b34f1f7af161(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:26:06,353 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04acb16af9bbc0bb9b137adb2146a06b: 2024-11-08T00:26:06,353 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., storeName=04acb16af9bbc0bb9b137adb2146a06b/C, priority=13, startTime=1731025566268; duration=0sec 2024-11-08T00:26:06,353 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:26:06,353 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04acb16af9bbc0bb9b137adb2146a06b:C 2024-11-08T00:26:06,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742417_1593 (size=12301) 2024-11-08T00:26:06,477 INFO [Thread-2280 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2280. 2024-11-08T00:26:06,477 INFO [Thread-2284 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2284. 2024-11-08T00:26:06,477 DEBUG [Thread-2284 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T00:26:06,477 DEBUG [Thread-2280 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T00:26:06,477 DEBUG [Thread-2284 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:26:06,477 DEBUG [Thread-2280 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:26:06,477 INFO [Thread-2276 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2276. 2024-11-08T00:26:06,477 DEBUG [Thread-2276 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T00:26:06,477 DEBUG [Thread-2280 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:26:06,477 DEBUG [Thread-2276 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:26:06,477 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:26:06,477 DEBUG [Thread-2276 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:26:06,477 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:26:06,479 DEBUG [Thread-2284 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:26:06,479 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:26:06,479 INFO [Thread-2282 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2282. 2024-11-08T00:26:06,479 DEBUG [Thread-2282 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T00:26:06,479 DEBUG [Thread-2282 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:26:06,479 DEBUG [Thread-2282 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:26:06,479 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:26:06,480 INFO [Thread-2278 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2278. 2024-11-08T00:26:06,480 DEBUG [Thread-2278 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T00:26:06,480 DEBUG [Thread-2278 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:26:06,480 DEBUG [Thread-2278 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:26:06,480 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:26:06,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-08T00:26:06,758 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/cb040097fd284283ad3dc128ccc6541f 2024-11-08T00:26:06,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/6ffbd2f270114743b703a1e219edaf08 is 50, key is test_row_0/B:col10/1731025565031/Put/seqid=0 2024-11-08T00:26:06,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742418_1594 (size=12301) 2024-11-08T00:26:06,938 INFO [Thread-2269 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2269. 2024-11-08T00:26:06,938 DEBUG [Thread-2269 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T00:26:06,938 INFO [Thread-2273 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2273. 2024-11-08T00:26:06,938 DEBUG [Thread-2269 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:26:06,939 DEBUG [Thread-2273 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T00:26:06,939 DEBUG [Thread-2273 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:26:06,939 DEBUG [Thread-2269 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:26:06,939 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:26:06,939 DEBUG [Thread-2273 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:26:06,939 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:26:06,939 INFO [Thread-2271 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2271. 2024-11-08T00:26:06,939 DEBUG [Thread-2271 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T00:26:06,939 DEBUG [Thread-2271 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:26:06,939 DEBUG [Thread-2271 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:26:06,939 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:26:07,169 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/6ffbd2f270114743b703a1e219edaf08 2024-11-08T00:26:07,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/333067b939cf4c7c97898a0046f577ad is 50, key is test_row_0/C:col10/1731025565031/Put/seqid=0 2024-11-08T00:26:07,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742419_1595 (size=12301) 2024-11-08T00:26:07,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. as already flushing 2024-11-08T00:26:07,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 04acb16af9bbc0bb9b137adb2146a06b 2024-11-08T00:26:07,199 INFO [Thread-2265 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2265. 2024-11-08T00:26:07,199 DEBUG [Thread-2265 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T00:26:07,199 DEBUG [Thread-2265 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:26:07,199 DEBUG [Thread-2265 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:26:07,199 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:26:07,209 INFO [Thread-2267 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2267. 2024-11-08T00:26:07,209 DEBUG [Thread-2267 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T00:26:07,209 DEBUG [Thread-2267 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:26:07,209 DEBUG [Thread-2267 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:26:07,209 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:26:07,576 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/333067b939cf4c7c97898a0046f577ad 2024-11-08T00:26:07,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/cb040097fd284283ad3dc128ccc6541f as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/cb040097fd284283ad3dc128ccc6541f 2024-11-08T00:26:07,581 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/cb040097fd284283ad3dc128ccc6541f, entries=150, sequenceid=381, filesize=12.0 K 2024-11-08T00:26:07,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/6ffbd2f270114743b703a1e219edaf08 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/6ffbd2f270114743b703a1e219edaf08 2024-11-08T00:26:07,584 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/6ffbd2f270114743b703a1e219edaf08, entries=150, sequenceid=381, filesize=12.0 K 2024-11-08T00:26:07,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/333067b939cf4c7c97898a0046f577ad as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/333067b939cf4c7c97898a0046f577ad 2024-11-08T00:26:07,587 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/333067b939cf4c7c97898a0046f577ad, entries=150, sequenceid=381, filesize=12.0 K 2024-11-08T00:26:07,587 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=33.54 KB/34350 for 04acb16af9bbc0bb9b137adb2146a06b in 1245ms, sequenceid=381, compaction requested=false 2024-11-08T00:26:07,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for 04acb16af9bbc0bb9b137adb2146a06b: 2024-11-08T00:26:07,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:26:07,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-08T00:26:07,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-08T00:26:07,589 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-08T00:26:07,589 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0080 sec 2024-11-08T00:26:07,591 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 2.0130 sec 2024-11-08T00:26:07,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-08T00:26:07,728 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:26:07,728 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-08T00:26:07,728 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-08T00:26:07,728 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-11-08T00:26:07,728 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-11-08T00:26:07,728 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-11-08T00:26:07,728 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-11-08T00:26:07,728 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-08T00:26:07,728 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-08T00:26:07,728 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1519 2024-11-08T00:26:07,728 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4554 rows 2024-11-08T00:26:07,728 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1512 2024-11-08T00:26:07,728 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4531 rows 2024-11-08T00:26:07,728 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1517 2024-11-08T00:26:07,728 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4546 rows 2024-11-08T00:26:07,728 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1514 2024-11-08T00:26:07,728 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4533 rows 2024-11-08T00:26:07,728 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1509 2024-11-08T00:26:07,728 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4525 rows 2024-11-08T00:26:07,728 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T00:26:07,728 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T00:26:07,728 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:26:07,728 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:26:07,728 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:26:07,729 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-08T00:26:07,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-08T00:26:07,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-08T00:26:07,731 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731025567731"}]},"ts":"1731025567731"} 2024-11-08T00:26:07,732 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-08T00:26:07,732 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-08T00:26:07,733 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-08T00:26:07,734 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=04acb16af9bbc0bb9b137adb2146a06b, UNASSIGN}] 2024-11-08T00:26:07,735 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=04acb16af9bbc0bb9b137adb2146a06b, UNASSIGN 2024-11-08T00:26:07,735 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=04acb16af9bbc0bb9b137adb2146a06b, regionState=CLOSING, regionLocation=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:07,737 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=04acb16af9bbc0bb9b137adb2146a06b, UNASSIGN because future has completed 2024-11-08T00:26:07,737 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-08T00:26:07,737 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure 04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856}] 2024-11-08T00:26:07,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-08T00:26:07,889 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(122): Close 04acb16af9bbc0bb9b137adb2146a06b 2024-11-08T00:26:07,889 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-08T00:26:07,889 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1722): Closing 04acb16af9bbc0bb9b137adb2146a06b, disabling compactions & flushes 2024-11-08T00:26:07,889 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:26:07,889 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:26:07,889 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. after waiting 0 ms 2024-11-08T00:26:07,889 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:26:07,889 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(2902): Flushing 04acb16af9bbc0bb9b137adb2146a06b 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-08T00:26:07,892 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/531186de6f37475196884d6c2eaee59f is 50, key is test_row_0/A:col10/1731025567208/Put/seqid=0 2024-11-08T00:26:07,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742420_1596 (size=9857) 2024-11-08T00:26:08,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-08T00:26:08,295 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/531186de6f37475196884d6c2eaee59f 2024-11-08T00:26:08,301 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/a966e94874444d678646c5decbc52cb7 is 50, key is test_row_0/B:col10/1731025567208/Put/seqid=0 2024-11-08T00:26:08,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742421_1597 (size=9857) 2024-11-08T00:26:08,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-08T00:26:08,704 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/a966e94874444d678646c5decbc52cb7 2024-11-08T00:26:08,709 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/c28d2bde6142437eb3dc991e4afc49b1 is 50, key is test_row_0/C:col10/1731025567208/Put/seqid=0 2024-11-08T00:26:08,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742422_1598 (size=9857) 2024-11-08T00:26:08,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-08T00:26:09,112 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/c28d2bde6142437eb3dc991e4afc49b1 2024-11-08T00:26:09,115 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/A/531186de6f37475196884d6c2eaee59f as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/531186de6f37475196884d6c2eaee59f 2024-11-08T00:26:09,117 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/531186de6f37475196884d6c2eaee59f, entries=100, sequenceid=390, filesize=9.6 K 2024-11-08T00:26:09,118 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/B/a966e94874444d678646c5decbc52cb7 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/a966e94874444d678646c5decbc52cb7 2024-11-08T00:26:09,120 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/a966e94874444d678646c5decbc52cb7, entries=100, sequenceid=390, filesize=9.6 K 2024-11-08T00:26:09,120 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/.tmp/C/c28d2bde6142437eb3dc991e4afc49b1 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/c28d2bde6142437eb3dc991e4afc49b1 2024-11-08T00:26:09,123 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/c28d2bde6142437eb3dc991e4afc49b1, entries=100, sequenceid=390, filesize=9.6 K 2024-11-08T00:26:09,124 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 04acb16af9bbc0bb9b137adb2146a06b in 1234ms, sequenceid=390, compaction requested=true 2024-11-08T00:26:09,124 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/e407d50c85d64bfc847fa9e4a48292a1, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/510ec65edb8848b0908a03c84a718e32, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/9eae889bf07a4633a8ab2fb9c695c310, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/944a138753394559981bcaa7a0ba48f1, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/02a59da620bf49208320f8d718f6655f, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/2a3ce38c06b1498890fc0ba76a3c7f81, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/2a91b4ec4cca482ebb42fd165ab957da, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/a081f2cb07f2461a8a01366fd61acdb5, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/0eabb0cffcd046f3975e1ebc51129a09, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/5c7d1f7e98d142feb074c44b023f4207, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/a761abec878842d3ac51f11995d54c99, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/6fb43869503842eb8db07ac146bc5b51, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/21609a4d5154401f99ebb9986fbc1025, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/659b74e8c28f4378b878701e6e9174d2, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/aa290aea6aa343d88c1bd42f4932955e, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/fcf1885585b443a3af6e9d0b1b183f9c, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/deae58b4bbf24023865d006902bdbbe7, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/e9115335fe2c4ca08327b82c30d5ee28, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/6a715ccc55d44efb833d8f03791c5ed5, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/b016c0f48e0c4103adceb5ddb9dfbb9f, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/a282ce52cbdb4480b83dc973003986ea, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/60ccf8011ce94a379edb2756f0c13df9, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/2027c35d198f4525b52f84113c505b55, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/1c38c3a4e2b54053af7dee1733a85c00, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/123af6a9160f49daaf635a88942c260c, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/2696f838495a4be9965772f25a82966f] to archive 2024-11-08T00:26:09,125 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T00:26:09,126 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/e407d50c85d64bfc847fa9e4a48292a1 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/e407d50c85d64bfc847fa9e4a48292a1 2024-11-08T00:26:09,126 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/510ec65edb8848b0908a03c84a718e32 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/510ec65edb8848b0908a03c84a718e32 2024-11-08T00:26:09,128 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/9eae889bf07a4633a8ab2fb9c695c310 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/9eae889bf07a4633a8ab2fb9c695c310 2024-11-08T00:26:09,129 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/944a138753394559981bcaa7a0ba48f1 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/944a138753394559981bcaa7a0ba48f1 2024-11-08T00:26:09,130 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/02a59da620bf49208320f8d718f6655f to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/02a59da620bf49208320f8d718f6655f 2024-11-08T00:26:09,131 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/2a3ce38c06b1498890fc0ba76a3c7f81 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/2a3ce38c06b1498890fc0ba76a3c7f81 2024-11-08T00:26:09,131 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/2a91b4ec4cca482ebb42fd165ab957da to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/2a91b4ec4cca482ebb42fd165ab957da 2024-11-08T00:26:09,132 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/a081f2cb07f2461a8a01366fd61acdb5 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/a081f2cb07f2461a8a01366fd61acdb5 2024-11-08T00:26:09,133 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/0eabb0cffcd046f3975e1ebc51129a09 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/0eabb0cffcd046f3975e1ebc51129a09 2024-11-08T00:26:09,133 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/5c7d1f7e98d142feb074c44b023f4207 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/5c7d1f7e98d142feb074c44b023f4207 2024-11-08T00:26:09,134 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/a761abec878842d3ac51f11995d54c99 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/a761abec878842d3ac51f11995d54c99 2024-11-08T00:26:09,135 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/6fb43869503842eb8db07ac146bc5b51 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/6fb43869503842eb8db07ac146bc5b51 2024-11-08T00:26:09,135 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/21609a4d5154401f99ebb9986fbc1025 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/21609a4d5154401f99ebb9986fbc1025 2024-11-08T00:26:09,136 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/659b74e8c28f4378b878701e6e9174d2 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/659b74e8c28f4378b878701e6e9174d2 2024-11-08T00:26:09,137 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/aa290aea6aa343d88c1bd42f4932955e to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/aa290aea6aa343d88c1bd42f4932955e 2024-11-08T00:26:09,137 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/fcf1885585b443a3af6e9d0b1b183f9c to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/fcf1885585b443a3af6e9d0b1b183f9c 2024-11-08T00:26:09,138 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/deae58b4bbf24023865d006902bdbbe7 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/deae58b4bbf24023865d006902bdbbe7 2024-11-08T00:26:09,139 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/e9115335fe2c4ca08327b82c30d5ee28 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/e9115335fe2c4ca08327b82c30d5ee28 2024-11-08T00:26:09,139 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/6a715ccc55d44efb833d8f03791c5ed5 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/6a715ccc55d44efb833d8f03791c5ed5 2024-11-08T00:26:09,140 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/b016c0f48e0c4103adceb5ddb9dfbb9f to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/b016c0f48e0c4103adceb5ddb9dfbb9f 2024-11-08T00:26:09,141 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/a282ce52cbdb4480b83dc973003986ea to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/a282ce52cbdb4480b83dc973003986ea 2024-11-08T00:26:09,141 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/60ccf8011ce94a379edb2756f0c13df9 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/60ccf8011ce94a379edb2756f0c13df9 2024-11-08T00:26:09,142 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/2027c35d198f4525b52f84113c505b55 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/2027c35d198f4525b52f84113c505b55 2024-11-08T00:26:09,142 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/1c38c3a4e2b54053af7dee1733a85c00 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/1c38c3a4e2b54053af7dee1733a85c00 2024-11-08T00:26:09,143 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/123af6a9160f49daaf635a88942c260c to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/123af6a9160f49daaf635a88942c260c 2024-11-08T00:26:09,144 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/2696f838495a4be9965772f25a82966f to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/2696f838495a4be9965772f25a82966f 2024-11-08T00:26:09,145 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/c30363cdfa1d4f61a2486afc9809abb6, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/72b4450e2a2148b3a9f6e4f3c8750ce4, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/aea9d526c6e24da4ba58189cd4ef8f39, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/4348b5d1d0584f628ea659f4359a84e8, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/93511ef195f849d89cc01fadf679b261, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/6e2232c58b61456ebd8320af0eeb4a69, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/5abd29aa3cb9484b9d1964042f8955cc, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/f68eb3ef3f34466aa8787e5dfaa879ff, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/67f0e42e280d4ce1a2906cbe8583f33f, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/389e7bc65d0d40649e5316c891fe8a4e, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/702cd2145f494d1887469204028ac6f7, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/54007788ab404ec7b77d737f7c2490da, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/a22a245055394828beab86a6a6680d61, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/a585705bff044a5381f1db737f0b3ca1, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/f35771ee0ec94ed9b4248199ac3fec5d, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/bcd05fd7ab314f158b2cccbd3c3d02ac, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/0699517e6eea4ef5a988c20ff3f008c4, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/36ec2e9335c349a9877d4df72f856788, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/8ddb0e063ad4442fbaecb4a80d78216f, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/f9f6b2c0bf2a4e46b6582ad2643920e7, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/665e99802f3d49018f0f47c70404f8da, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/1d37d0f29c7f4ad88fbb676f2fc50d47, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/fda93b2578824b34b0e458553d5616a0, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/491a1a9402334a658cbfa0443dcd0a6d, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/a2159f1ac90f41b18482ffa2684ebf3c, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/400e103bd272405aab6d358cb0450b68] to archive 2024-11-08T00:26:09,145 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T00:26:09,146 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/c30363cdfa1d4f61a2486afc9809abb6 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/c30363cdfa1d4f61a2486afc9809abb6 2024-11-08T00:26:09,147 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/72b4450e2a2148b3a9f6e4f3c8750ce4 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/72b4450e2a2148b3a9f6e4f3c8750ce4 2024-11-08T00:26:09,148 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/aea9d526c6e24da4ba58189cd4ef8f39 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/aea9d526c6e24da4ba58189cd4ef8f39 2024-11-08T00:26:09,148 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/4348b5d1d0584f628ea659f4359a84e8 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/4348b5d1d0584f628ea659f4359a84e8 2024-11-08T00:26:09,149 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/93511ef195f849d89cc01fadf679b261 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/93511ef195f849d89cc01fadf679b261 2024-11-08T00:26:09,150 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/6e2232c58b61456ebd8320af0eeb4a69 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/6e2232c58b61456ebd8320af0eeb4a69 2024-11-08T00:26:09,150 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/5abd29aa3cb9484b9d1964042f8955cc to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/5abd29aa3cb9484b9d1964042f8955cc 2024-11-08T00:26:09,151 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/f68eb3ef3f34466aa8787e5dfaa879ff to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/f68eb3ef3f34466aa8787e5dfaa879ff 2024-11-08T00:26:09,152 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/67f0e42e280d4ce1a2906cbe8583f33f to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/67f0e42e280d4ce1a2906cbe8583f33f 2024-11-08T00:26:09,153 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/389e7bc65d0d40649e5316c891fe8a4e to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/389e7bc65d0d40649e5316c891fe8a4e 2024-11-08T00:26:09,154 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/702cd2145f494d1887469204028ac6f7 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/702cd2145f494d1887469204028ac6f7 2024-11-08T00:26:09,154 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/54007788ab404ec7b77d737f7c2490da to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/54007788ab404ec7b77d737f7c2490da 2024-11-08T00:26:09,155 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/a22a245055394828beab86a6a6680d61 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/a22a245055394828beab86a6a6680d61 2024-11-08T00:26:09,156 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/a585705bff044a5381f1db737f0b3ca1 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/a585705bff044a5381f1db737f0b3ca1 2024-11-08T00:26:09,156 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/f35771ee0ec94ed9b4248199ac3fec5d to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/f35771ee0ec94ed9b4248199ac3fec5d 2024-11-08T00:26:09,157 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/bcd05fd7ab314f158b2cccbd3c3d02ac to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/bcd05fd7ab314f158b2cccbd3c3d02ac 2024-11-08T00:26:09,158 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/0699517e6eea4ef5a988c20ff3f008c4 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/0699517e6eea4ef5a988c20ff3f008c4 2024-11-08T00:26:09,159 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/36ec2e9335c349a9877d4df72f856788 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/36ec2e9335c349a9877d4df72f856788 2024-11-08T00:26:09,160 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/8ddb0e063ad4442fbaecb4a80d78216f to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/8ddb0e063ad4442fbaecb4a80d78216f 2024-11-08T00:26:09,160 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/f9f6b2c0bf2a4e46b6582ad2643920e7 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/f9f6b2c0bf2a4e46b6582ad2643920e7 2024-11-08T00:26:09,161 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/665e99802f3d49018f0f47c70404f8da to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/665e99802f3d49018f0f47c70404f8da 2024-11-08T00:26:09,162 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/1d37d0f29c7f4ad88fbb676f2fc50d47 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/1d37d0f29c7f4ad88fbb676f2fc50d47 2024-11-08T00:26:09,163 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/fda93b2578824b34b0e458553d5616a0 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/fda93b2578824b34b0e458553d5616a0 2024-11-08T00:26:09,163 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/491a1a9402334a658cbfa0443dcd0a6d to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/491a1a9402334a658cbfa0443dcd0a6d 2024-11-08T00:26:09,164 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/a2159f1ac90f41b18482ffa2684ebf3c to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/a2159f1ac90f41b18482ffa2684ebf3c 2024-11-08T00:26:09,165 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/400e103bd272405aab6d358cb0450b68 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/400e103bd272405aab6d358cb0450b68 2024-11-08T00:26:09,166 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/66c27b728d084ebda6ad9c8577f694f7, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/d8f7538b6cba497f9b87a374193e04ec, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/e17c34434ee3451fa139e1b2f5d6b330, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/80c1d6450cf544158213f63cc6c74e0a, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/db94ffb1f6dc4008884a8e9100938ee3, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/ac8b221a70f14dfea79ec35c80b377d6, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/927fb6fa175348f38547ea2f08049ccf, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/87e3b2c9b7244c29bfcf68e4c75f6894, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/9e2b0c9399804c90835bb651b7a638c4, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/a98c2637756d45d983157fd7dbe90dc6, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/94d5cd355f3f49a1aee9ac9af6b1adae, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/110dcc6900f44949a53d3825bc885c2b, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/abd1d452fc2146fca43432fa4d820585, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/f57a2f514e304a7db7624dabcd63c49a, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/86149fec8657483f99c23f03cd869246, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/8bbd663759b54b8ab21d59af15e1557b, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/14e8bba52b744775b1c6126393fe9853, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/9c3795acc0c14d56a33e4ceefe2805f9, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/c564f830945848dd8951c0c12cdc106b, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/c43005bfa65d41c7ba04c9084a97b12f, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/26f16580c114477ca2585df28a745895, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/bc81c62175fc4449a19b5378cedf3a4c, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/2db28c87fbcf4b06a139c2a530ec2985, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/8705030c5f5c4c7ea3e73344af383238, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/cab46df99a84441da6cd2c21c781e2cf, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/a3f9f277852a4ece9e67eda74262f815] to archive 2024-11-08T00:26:09,167 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T00:26:09,168 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/66c27b728d084ebda6ad9c8577f694f7 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/66c27b728d084ebda6ad9c8577f694f7 2024-11-08T00:26:09,169 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/d8f7538b6cba497f9b87a374193e04ec to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/d8f7538b6cba497f9b87a374193e04ec 2024-11-08T00:26:09,170 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/e17c34434ee3451fa139e1b2f5d6b330 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/e17c34434ee3451fa139e1b2f5d6b330 2024-11-08T00:26:09,171 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/80c1d6450cf544158213f63cc6c74e0a to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/80c1d6450cf544158213f63cc6c74e0a 2024-11-08T00:26:09,171 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/db94ffb1f6dc4008884a8e9100938ee3 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/db94ffb1f6dc4008884a8e9100938ee3 2024-11-08T00:26:09,172 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/ac8b221a70f14dfea79ec35c80b377d6 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/ac8b221a70f14dfea79ec35c80b377d6 2024-11-08T00:26:09,173 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/927fb6fa175348f38547ea2f08049ccf to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/927fb6fa175348f38547ea2f08049ccf 2024-11-08T00:26:09,174 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/87e3b2c9b7244c29bfcf68e4c75f6894 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/87e3b2c9b7244c29bfcf68e4c75f6894 2024-11-08T00:26:09,174 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/9e2b0c9399804c90835bb651b7a638c4 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/9e2b0c9399804c90835bb651b7a638c4 2024-11-08T00:26:09,175 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/a98c2637756d45d983157fd7dbe90dc6 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/a98c2637756d45d983157fd7dbe90dc6 2024-11-08T00:26:09,176 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/94d5cd355f3f49a1aee9ac9af6b1adae to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/94d5cd355f3f49a1aee9ac9af6b1adae 2024-11-08T00:26:09,176 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/110dcc6900f44949a53d3825bc885c2b to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/110dcc6900f44949a53d3825bc885c2b 2024-11-08T00:26:09,177 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/abd1d452fc2146fca43432fa4d820585 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/abd1d452fc2146fca43432fa4d820585 2024-11-08T00:26:09,178 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/f57a2f514e304a7db7624dabcd63c49a to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/f57a2f514e304a7db7624dabcd63c49a 2024-11-08T00:26:09,178 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/86149fec8657483f99c23f03cd869246 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/86149fec8657483f99c23f03cd869246 2024-11-08T00:26:09,179 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/8bbd663759b54b8ab21d59af15e1557b to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/8bbd663759b54b8ab21d59af15e1557b 2024-11-08T00:26:09,179 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/14e8bba52b744775b1c6126393fe9853 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/14e8bba52b744775b1c6126393fe9853 2024-11-08T00:26:09,180 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/9c3795acc0c14d56a33e4ceefe2805f9 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/9c3795acc0c14d56a33e4ceefe2805f9 2024-11-08T00:26:09,181 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/c564f830945848dd8951c0c12cdc106b to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/c564f830945848dd8951c0c12cdc106b 2024-11-08T00:26:09,182 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/c43005bfa65d41c7ba04c9084a97b12f to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/c43005bfa65d41c7ba04c9084a97b12f 2024-11-08T00:26:09,182 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/26f16580c114477ca2585df28a745895 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/26f16580c114477ca2585df28a745895 2024-11-08T00:26:09,183 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/bc81c62175fc4449a19b5378cedf3a4c to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/bc81c62175fc4449a19b5378cedf3a4c 2024-11-08T00:26:09,184 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/2db28c87fbcf4b06a139c2a530ec2985 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/2db28c87fbcf4b06a139c2a530ec2985 2024-11-08T00:26:09,185 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/8705030c5f5c4c7ea3e73344af383238 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/8705030c5f5c4c7ea3e73344af383238 2024-11-08T00:26:09,186 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/cab46df99a84441da6cd2c21c781e2cf to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/cab46df99a84441da6cd2c21c781e2cf 2024-11-08T00:26:09,186 DEBUG [StoreCloser-TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/a3f9f277852a4ece9e67eda74262f815 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/a3f9f277852a4ece9e67eda74262f815 2024-11-08T00:26:09,190 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/recovered.edits/393.seqid, newMaxSeqId=393, maxSeqId=1 2024-11-08T00:26:09,191 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. 2024-11-08T00:26:09,191 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1676): Region close journal for 04acb16af9bbc0bb9b137adb2146a06b: Waiting for close lock at 1731025567889Running coprocessor pre-close hooks at 1731025567889Disabling compacts and flushes for region at 1731025567889Disabling writes for close at 1731025567889Obtaining lock to block concurrent updates at 1731025567889Preparing flush snapshotting stores in 04acb16af9bbc0bb9b137adb2146a06b at 1731025567889Finished memstore snapshotting TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731025567889Flushing stores of TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b. at 1731025567890 (+1 ms)Flushing 04acb16af9bbc0bb9b137adb2146a06b/A: creating writer at 1731025567890Flushing 04acb16af9bbc0bb9b137adb2146a06b/A: appending metadata at 1731025567892 (+2 ms)Flushing 04acb16af9bbc0bb9b137adb2146a06b/A: closing flushed file at 1731025567892Flushing 04acb16af9bbc0bb9b137adb2146a06b/B: creating writer at 1731025568298 (+406 ms)Flushing 04acb16af9bbc0bb9b137adb2146a06b/B: appending metadata at 1731025568300 (+2 ms)Flushing 04acb16af9bbc0bb9b137adb2146a06b/B: closing flushed file at 1731025568300Flushing 04acb16af9bbc0bb9b137adb2146a06b/C: creating writer at 1731025568707 (+407 ms)Flushing 04acb16af9bbc0bb9b137adb2146a06b/C: appending metadata at 1731025568708 (+1 ms)Flushing 04acb16af9bbc0bb9b137adb2146a06b/C: closing flushed file at 1731025568708Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e99f517: reopening flushed file at 1731025569114 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2af8c738: reopening flushed file at 1731025569117 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4d735e63: reopening flushed file at 1731025569120 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 04acb16af9bbc0bb9b137adb2146a06b in 1234ms, sequenceid=390, compaction requested=true at 1731025569124 (+4 ms)Writing region close event to WAL at 1731025569187 (+63 ms)Running coprocessor post-close hooks at 1731025569190 (+3 ms)Closed at 1731025569190 2024-11-08T00:26:09,193 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(157): Closed 04acb16af9bbc0bb9b137adb2146a06b 2024-11-08T00:26:09,193 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=04acb16af9bbc0bb9b137adb2146a06b, regionState=CLOSED 2024-11-08T00:26:09,194 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure 04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 because future has completed 2024-11-08T00:26:09,197 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-08T00:26:09,197 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseRegionProcedure 04acb16af9bbc0bb9b137adb2146a06b, server=3302f0f507bd,40479,1731025415856 in 1.4580 sec 2024-11-08T00:26:09,198 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-08T00:26:09,198 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=04acb16af9bbc0bb9b137adb2146a06b, UNASSIGN in 1.4630 sec 2024-11-08T00:26:09,200 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-08T00:26:09,200 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4660 sec 2024-11-08T00:26:09,202 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731025569202"}]},"ts":"1731025569202"} 2024-11-08T00:26:09,203 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-08T00:26:09,203 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-08T00:26:09,205 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4750 sec 2024-11-08T00:26:09,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-08T00:26:09,887 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:26:09,888 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-08T00:26:09,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T00:26:09,889 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=143, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T00:26:09,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-08T00:26:09,889 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=143, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T00:26:09,891 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b 2024-11-08T00:26:09,892 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A, FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B, FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C, FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/recovered.edits] 2024-11-08T00:26:09,894 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/394b4767d94a48fe951ae57a050df01f to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/394b4767d94a48fe951ae57a050df01f 2024-11-08T00:26:09,895 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/531186de6f37475196884d6c2eaee59f to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/531186de6f37475196884d6c2eaee59f 2024-11-08T00:26:09,896 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/cb040097fd284283ad3dc128ccc6541f to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/A/cb040097fd284283ad3dc128ccc6541f 2024-11-08T00:26:09,897 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/1791701f8a8c4219a0f6f8cd569082df to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/1791701f8a8c4219a0f6f8cd569082df 2024-11-08T00:26:09,898 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/6ffbd2f270114743b703a1e219edaf08 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/6ffbd2f270114743b703a1e219edaf08 2024-11-08T00:26:09,899 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/a966e94874444d678646c5decbc52cb7 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/B/a966e94874444d678646c5decbc52cb7 2024-11-08T00:26:09,900 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/333067b939cf4c7c97898a0046f577ad to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/333067b939cf4c7c97898a0046f577ad 2024-11-08T00:26:09,901 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/c28d2bde6142437eb3dc991e4afc49b1 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/c28d2bde6142437eb3dc991e4afc49b1 2024-11-08T00:26:09,902 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/eb0a388c46544962a7a3b34f1f7af161 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/C/eb0a388c46544962a7a3b34f1f7af161 2024-11-08T00:26:09,904 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/recovered.edits/393.seqid to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b/recovered.edits/393.seqid 2024-11-08T00:26:09,904 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/04acb16af9bbc0bb9b137adb2146a06b 2024-11-08T00:26:09,904 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-08T00:26:09,906 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=143, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T00:26:09,908 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-08T00:26:09,909 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-08T00:26:09,910 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=143, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T00:26:09,910 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-08T00:26:09,910 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731025569910"}]},"ts":"9223372036854775807"} 2024-11-08T00:26:09,912 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-08T00:26:09,912 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 04acb16af9bbc0bb9b137adb2146a06b, NAME => 'TestAcidGuarantees,,1731025544281.04acb16af9bbc0bb9b137adb2146a06b.', STARTKEY => '', ENDKEY => ''}] 2024-11-08T00:26:09,912 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-08T00:26:09,912 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731025569912"}]},"ts":"9223372036854775807"} 2024-11-08T00:26:09,913 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-08T00:26:09,914 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=143, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T00:26:09,915 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 26 msec 2024-11-08T00:26:09,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-08T00:26:09,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-08T00:26:09,999 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:26:10,005 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=240 (was 238) - Thread LEAK? -, OpenFileDescriptor=447 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=591 (was 666), ProcessCount=11 (was 12), AvailableMemoryMB=6385 (was 6425) 2024-11-08T00:26:10,013 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=240, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=591, ProcessCount=11, AvailableMemoryMB=6385 2024-11-08T00:26:10,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-08T00:26:10,014 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T00:26:10,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-08T00:26:10,016 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T00:26:10,016 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:10,016 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 144 2024-11-08T00:26:10,017 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T00:26:10,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-08T00:26:10,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742423_1599 (size=959) 2024-11-08T00:26:10,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-08T00:26:10,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-08T00:26:10,423 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f 2024-11-08T00:26:10,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742424_1600 (size=53) 2024-11-08T00:26:10,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-08T00:26:10,830 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:26:10,830 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 22b286596054a5e4265c6f73c6acf0d1, disabling compactions & flushes 2024-11-08T00:26:10,830 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:10,830 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:10,830 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. after waiting 0 ms 2024-11-08T00:26:10,830 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:10,830 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:10,830 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 22b286596054a5e4265c6f73c6acf0d1: Waiting for close lock at 1731025570830Disabling compacts and flushes for region at 1731025570830Disabling writes for close at 1731025570830Writing region close event to WAL at 1731025570830Closed at 1731025570830 2024-11-08T00:26:10,831 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T00:26:10,832 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731025570831"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731025570831"}]},"ts":"1731025570831"} 2024-11-08T00:26:10,833 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-08T00:26:10,833 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T00:26:10,833 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731025570833"}]},"ts":"1731025570833"} 2024-11-08T00:26:10,834 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-08T00:26:10,834 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=22b286596054a5e4265c6f73c6acf0d1, ASSIGN}] 2024-11-08T00:26:10,835 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=22b286596054a5e4265c6f73c6acf0d1, ASSIGN 2024-11-08T00:26:10,836 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=22b286596054a5e4265c6f73c6acf0d1, ASSIGN; state=OFFLINE, location=3302f0f507bd,40479,1731025415856; forceNewPlan=false, retain=false 2024-11-08T00:26:10,986 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=22b286596054a5e4265c6f73c6acf0d1, regionState=OPENING, regionLocation=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:10,988 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=22b286596054a5e4265c6f73c6acf0d1, ASSIGN because future has completed 2024-11-08T00:26:10,988 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure 22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856}] 2024-11-08T00:26:11,143 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:11,143 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7752): Opening region: {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} 2024-11-08T00:26:11,143 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:11,143 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:26:11,144 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7794): checking encryption for 22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:11,144 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7797): checking classloading for 22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:11,145 INFO [StoreOpener-22b286596054a5e4265c6f73c6acf0d1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:11,146 INFO [StoreOpener-22b286596054a5e4265c6f73c6acf0d1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 22b286596054a5e4265c6f73c6acf0d1 columnFamilyName A 2024-11-08T00:26:11,146 DEBUG [StoreOpener-22b286596054a5e4265c6f73c6acf0d1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:11,146 INFO [StoreOpener-22b286596054a5e4265c6f73c6acf0d1-1 {}] regionserver.HStore(327): Store=22b286596054a5e4265c6f73c6acf0d1/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:26:11,146 INFO [StoreOpener-22b286596054a5e4265c6f73c6acf0d1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:11,147 INFO [StoreOpener-22b286596054a5e4265c6f73c6acf0d1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 22b286596054a5e4265c6f73c6acf0d1 columnFamilyName B 2024-11-08T00:26:11,147 DEBUG [StoreOpener-22b286596054a5e4265c6f73c6acf0d1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:11,147 INFO [StoreOpener-22b286596054a5e4265c6f73c6acf0d1-1 {}] regionserver.HStore(327): Store=22b286596054a5e4265c6f73c6acf0d1/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:26:11,147 INFO [StoreOpener-22b286596054a5e4265c6f73c6acf0d1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:11,148 INFO [StoreOpener-22b286596054a5e4265c6f73c6acf0d1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 22b286596054a5e4265c6f73c6acf0d1 columnFamilyName C 2024-11-08T00:26:11,148 DEBUG [StoreOpener-22b286596054a5e4265c6f73c6acf0d1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:11,149 INFO [StoreOpener-22b286596054a5e4265c6f73c6acf0d1-1 {}] regionserver.HStore(327): Store=22b286596054a5e4265c6f73c6acf0d1/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:26:11,149 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1038): replaying wal for 22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:11,149 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:11,150 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:11,150 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1048): stopping wal replay for 22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:11,150 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1060): Cleaning up temporary data for 22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:11,150 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-08T00:26:11,151 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1093): writing seq id for 22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:11,152 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T00:26:11,153 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1114): Opened 22b286596054a5e4265c6f73c6acf0d1; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73929076, jitterRate=0.10162907838821411}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-08T00:26:11,153 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:11,153 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1006): Region open journal for 22b286596054a5e4265c6f73c6acf0d1: Running coprocessor pre-open hook at 1731025571144Writing region info on filesystem at 1731025571144Initializing all the Stores at 1731025571144Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731025571144Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731025571145 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731025571145Cleaning up temporary data from old regions at 1731025571150 (+5 ms)Running coprocessor post-open hooks at 1731025571153 (+3 ms)Region opened successfully at 1731025571153 2024-11-08T00:26:11,154 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., pid=146, masterSystemTime=1731025571140 2024-11-08T00:26:11,155 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:11,155 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:11,156 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=22b286596054a5e4265c6f73c6acf0d1, regionState=OPEN, openSeqNum=2, regionLocation=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:11,157 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=146, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure 22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 because future has completed 2024-11-08T00:26:11,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-08T00:26:11,158 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-08T00:26:11,158 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; OpenRegionProcedure 22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 in 169 msec 2024-11-08T00:26:11,160 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-08T00:26:11,160 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=22b286596054a5e4265c6f73c6acf0d1, ASSIGN in 324 msec 2024-11-08T00:26:11,160 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T00:26:11,160 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731025571160"}]},"ts":"1731025571160"} 2024-11-08T00:26:11,162 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-08T00:26:11,162 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T00:26:11,163 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1480 sec 2024-11-08T00:26:12,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-08T00:26:12,177 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:26:12,178 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ab36c11, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:26:12,178 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:26:12,178 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:26:12,179 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:26:12,179 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:26:12,179 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:26:12,179 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f187aa5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:26:12,179 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:26:12,180 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:26:12,180 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:26:12,180 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36164, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:26:12,181 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@491540f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:26:12,181 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:26:12,182 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:26:12,182 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:26:12,182 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45970, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:26:12,183 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-08T00:26:12,184 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 3302f0f507bd,36785,1731025414356 2024-11-08T00:26:12,184 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5be62c04 2024-11-08T00:26:12,184 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-08T00:26:12,185 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36166, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-08T00:26:12,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-08T00:26:12,186 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T00:26:12,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=147, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-08T00:26:12,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742425_1601 (size=995) 2024-11-08T00:26:12,595 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-08T00:26:12,595 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-08T00:26:12,597 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-08T00:26:12,598 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=22b286596054a5e4265c6f73c6acf0d1, REOPEN/MOVE}] 2024-11-08T00:26:12,599 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=22b286596054a5e4265c6f73c6acf0d1, REOPEN/MOVE 2024-11-08T00:26:12,599 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=22b286596054a5e4265c6f73c6acf0d1, regionState=CLOSING, regionLocation=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:12,600 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=22b286596054a5e4265c6f73c6acf0d1, REOPEN/MOVE because future has completed 2024-11-08T00:26:12,600 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-08T00:26:12,600 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure 22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856}] 2024-11-08T00:26:12,752 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(122): Close 22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:12,752 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-08T00:26:12,752 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1722): Closing 22b286596054a5e4265c6f73c6acf0d1, disabling compactions & flushes 2024-11-08T00:26:12,752 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:12,753 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:12,753 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. after waiting 0 ms 2024-11-08T00:26:12,753 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:12,755 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T00:26:12,756 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:12,756 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1676): Region close journal for 22b286596054a5e4265c6f73c6acf0d1: Waiting for close lock at 1731025572752Running coprocessor pre-close hooks at 1731025572752Disabling compacts and flushes for region at 1731025572752Disabling writes for close at 1731025572753 (+1 ms)Writing region close event to WAL at 1731025572753Running coprocessor post-close hooks at 1731025572756 (+3 ms)Closed at 1731025572756 2024-11-08T00:26:12,756 WARN [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegionServer(3299): Not adding moved region record: 22b286596054a5e4265c6f73c6acf0d1 to self. 2024-11-08T00:26:12,757 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(157): Closed 22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:12,758 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=22b286596054a5e4265c6f73c6acf0d1, regionState=CLOSED 2024-11-08T00:26:12,758 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure 22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 because future has completed 2024-11-08T00:26:12,760 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-08T00:26:12,760 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; CloseRegionProcedure 22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 in 159 msec 2024-11-08T00:26:12,760 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=22b286596054a5e4265c6f73c6acf0d1, REOPEN/MOVE; state=CLOSED, location=3302f0f507bd,40479,1731025415856; forceNewPlan=false, retain=true 2024-11-08T00:26:12,911 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=22b286596054a5e4265c6f73c6acf0d1, regionState=OPENING, regionLocation=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:12,912 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=22b286596054a5e4265c6f73c6acf0d1, REOPEN/MOVE because future has completed 2024-11-08T00:26:12,913 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure 22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856}] 2024-11-08T00:26:13,066 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:13,067 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7752): Opening region: {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} 2024-11-08T00:26:13,067 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:13,067 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T00:26:13,067 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7794): checking encryption for 22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:13,067 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7797): checking classloading for 22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:13,068 INFO [StoreOpener-22b286596054a5e4265c6f73c6acf0d1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:13,069 INFO [StoreOpener-22b286596054a5e4265c6f73c6acf0d1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 22b286596054a5e4265c6f73c6acf0d1 columnFamilyName A 2024-11-08T00:26:13,070 DEBUG [StoreOpener-22b286596054a5e4265c6f73c6acf0d1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:13,070 INFO [StoreOpener-22b286596054a5e4265c6f73c6acf0d1-1 {}] regionserver.HStore(327): Store=22b286596054a5e4265c6f73c6acf0d1/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:26:13,070 INFO [StoreOpener-22b286596054a5e4265c6f73c6acf0d1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:13,071 INFO [StoreOpener-22b286596054a5e4265c6f73c6acf0d1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 22b286596054a5e4265c6f73c6acf0d1 columnFamilyName B 2024-11-08T00:26:13,071 DEBUG [StoreOpener-22b286596054a5e4265c6f73c6acf0d1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:13,071 INFO [StoreOpener-22b286596054a5e4265c6f73c6acf0d1-1 {}] regionserver.HStore(327): Store=22b286596054a5e4265c6f73c6acf0d1/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:26:13,071 INFO [StoreOpener-22b286596054a5e4265c6f73c6acf0d1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:13,072 INFO [StoreOpener-22b286596054a5e4265c6f73c6acf0d1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 22b286596054a5e4265c6f73c6acf0d1 columnFamilyName C 2024-11-08T00:26:13,072 DEBUG [StoreOpener-22b286596054a5e4265c6f73c6acf0d1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:13,072 INFO [StoreOpener-22b286596054a5e4265c6f73c6acf0d1-1 {}] regionserver.HStore(327): Store=22b286596054a5e4265c6f73c6acf0d1/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T00:26:13,072 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1038): replaying wal for 22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:13,073 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:13,074 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:13,074 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1048): stopping wal replay for 22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:13,074 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1060): Cleaning up temporary data for 22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:13,075 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-08T00:26:13,076 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1093): writing seq id for 22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:13,076 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1114): Opened 22b286596054a5e4265c6f73c6acf0d1; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71240684, jitterRate=0.06156891584396362}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-08T00:26:13,076 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:13,077 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1006): Region open journal for 22b286596054a5e4265c6f73c6acf0d1: Running coprocessor pre-open hook at 1731025573067Writing region info on filesystem at 1731025573067Initializing all the Stores at 1731025573068 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731025573068Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731025573068Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731025573068Cleaning up temporary data from old regions at 1731025573074 (+6 ms)Running coprocessor post-open hooks at 1731025573076 (+2 ms)Region opened successfully at 1731025573077 (+1 ms) 2024-11-08T00:26:13,077 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., pid=151, masterSystemTime=1731025573064 2024-11-08T00:26:13,079 DEBUG [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:13,079 INFO [RS_OPEN_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:13,079 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=22b286596054a5e4265c6f73c6acf0d1, regionState=OPEN, openSeqNum=5, regionLocation=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:13,080 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure 22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 because future has completed 2024-11-08T00:26:13,082 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=149 2024-11-08T00:26:13,082 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=149, state=SUCCESS, hasLock=false; OpenRegionProcedure 22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 in 168 msec 2024-11-08T00:26:13,084 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-08T00:26:13,084 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=22b286596054a5e4265c6f73c6acf0d1, REOPEN/MOVE in 484 msec 2024-11-08T00:26:13,086 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-08T00:26:13,086 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 487 msec 2024-11-08T00:26:13,087 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 900 msec 2024-11-08T00:26:13,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-08T00:26:13,088 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:26:13,089 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59c68a61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:26:13,089 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:26:13,089 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:26:13,090 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:26:13,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:26:13,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:26:13,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36fa8f99, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:26:13,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:26:13,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:26:13,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:26:13,091 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36182, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:26:13,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52736bbc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:26:13,092 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5aeb1c22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:26:13,092 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:26:13,092 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:26:13,093 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:26:13,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:26:13,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:26:13,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c62a11f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:26:13,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:26:13,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:26:13,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:26:13,094 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36204, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:26:13,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46f6be2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:26:13,095 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@193a5b68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:26:13,095 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:26:13,095 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:26:13,095 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:26:13,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:26:13,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:26:13,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3004f82e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:26:13,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:26:13,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:26:13,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:26:13,097 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36228, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:26:13,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41a75663, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:26:13,098 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c20cfe4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:26:13,098 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:26:13,098 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:26:13,099 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:26:13,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:26:13,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:26:13,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33b87363, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:26:13,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:26:13,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:26:13,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:26:13,100 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36238, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:26:13,100 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e10bb3a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:26:13,101 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d0eef11, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:26:13,101 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:26:13,101 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:26:13,101 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:26:13,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:26:13,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:26:13,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@508bbb58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:26:13,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:26:13,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:26:13,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:26:13,103 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36270, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:26:13,103 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@763a296d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:26:13,104 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@385c264c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:26:13,104 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:26:13,104 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:26:13,104 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:26:13,104 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:26:13,105 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:26:13,105 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2323d52a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:26:13,105 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:26:13,105 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:26:13,105 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:26:13,106 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36286, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:26:13,106 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@89b610a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:26:13,106 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a06379d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:26:13,106 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:26:13,107 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:26:13,107 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:26:13,107 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:26:13,107 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:26:13,107 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d9841c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:26:13,107 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:26:13,108 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:26:13,108 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:26:13,108 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36296, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:26:13,109 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cfc80da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:26:13,109 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b2b0540, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:26:13,109 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:26:13,109 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:26:13,110 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:26:13,110 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:26:13,110 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:26:13,110 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74ec3cd8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:26:13,110 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:26:13,110 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:26:13,110 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:26:13,111 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36308, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:26:13,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66c2fcb4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:26:13,112 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e447c48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:26:13,112 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:26:13,112 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:26:13,113 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:26:13,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:26:13,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:26:13,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5179b6c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:26:13,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:26:13,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:26:13,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:26:13,114 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36318, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:26:13,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@448d6024, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:26:13,115 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ba5f0ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:26:13,115 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3302f0f507bd,36785,-1 for getting cluster id 2024-11-08T00:26:13,115 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T00:26:13,116 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3187367b-61bb-4afc-a729-21b60de76def' 2024-11-08T00:26:13,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T00:26:13,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3187367b-61bb-4afc-a729-21b60de76def" 2024-11-08T00:26:13,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@530a19d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:26:13,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3302f0f507bd,36785,-1] 2024-11-08T00:26:13,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T00:26:13,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:26:13,117 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36336, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T00:26:13,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25cfb938, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T00:26:13,118 DEBUG [Thread-2659 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:26:13,118 DEBUG [Thread-2668 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:26:13,118 DEBUG [Thread-2663 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:26:13,118 DEBUG [Thread-2657 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:26:13,118 DEBUG [Thread-2674 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:26:13,119 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-08T00:26:13,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:26:13,119 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:26:13,119 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:26:13,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:26:13,119 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:26:13,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:26:13,119 DEBUG [Thread-2666 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:26:13,119 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:26:13,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:26:13,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:26:13,119 DEBUG [Thread-2670 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:26:13,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:26:13,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-08T00:26:13,120 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45976, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:26:13,120 DEBUG [Thread-2655 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:26:13,120 DEBUG [Thread-2672 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:26:13,120 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:26:13,120 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45986, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:26:13,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:26:13,121 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46004, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:26:13,121 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:26:13,121 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45996, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:26:13,121 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:26:13,121 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46020, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:26:13,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5] 2024-11-08T00:26:13,122 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46038, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:26:13,122 DEBUG [Thread-2661 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T00:26:13,122 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T00:26:13,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:26:13,122 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46030, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:26:13,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:26:13,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-08T00:26:13,123 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:26:13,123 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:26:13,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5] 2024-11-08T00:26:13,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5] 2024-11-08T00:26:13,123 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T00:26:13,123 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T00:26:13,123 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5] 2024-11-08T00:26:13,123 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5] 2024-11-08T00:26:13,124 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46048, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:26:13,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5] 2024-11-08T00:26:13,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3302f0f507bd,40479,1731025415856, seqNum=-1] 2024-11-08T00:26:13,124 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46042, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:26:13,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T00:26:13,125 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5] 2024-11-08T00:26:13,125 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46052, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T00:26:13,126 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5] 2024-11-08T00:26:13,127 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5] 2024-11-08T00:26:13,128 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5] 2024-11-08T00:26:13,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:13,133 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:26:13,133 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 22b286596054a5e4265c6f73c6acf0d1 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-08T00:26:13,160 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411085e6e1853ba104019be4d18403ec02423_22b286596054a5e4265c6f73c6acf0d1 is 50, key is test_row_0/A:col10/1731025573133/Put/seqid=0 2024-11-08T00:26:13,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:13,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:13,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025633156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:13,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025633157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:13,162 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:13,162 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:13,162 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:13,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:13,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:13,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:13,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:13,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025633158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:13,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:13,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:13,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:13,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:13,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025633158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:13,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:13,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:13,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:13,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025633158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:13,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:13,164 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:13,164 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:13,164 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:13,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742426_1602 (size=12154) 2024-11-08T00:26:13,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-08T00:26:13,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:13,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025633267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:13,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:13,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:13,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:13,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:13,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025633267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:13,269 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:13,269 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:13,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:13,269 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:13,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025633267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:13,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:13,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:13,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:13,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:13,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:13,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025633267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:13,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025633267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:13,269 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:13,269 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:13,269 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:13,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:13,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:13,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:13,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-08T00:26:13,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:13,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. as already flushing 2024-11-08T00:26:13,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:13,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:13,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:13,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:13,431 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-08T00:26:13,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:13,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. as already flushing 2024-11-08T00:26:13,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:13,432 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:13,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:13,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:13,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-08T00:26:13,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:13,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025633477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:13,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:13,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:13,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:13,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:13,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025633477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:13,479 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:13,479 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:13,479 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:13,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:13,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025633477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:13,480 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:13,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:13,480 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:13,480 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:13,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025633477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:13,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:13,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:13,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:13,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:13,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025633477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:13,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:13,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:13,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:13,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-08T00:26:13,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:13,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. as already flushing 2024-11-08T00:26:13,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:13,585 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:13,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:13,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:13,593 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:13,596 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411085e6e1853ba104019be4d18403ec02423_22b286596054a5e4265c6f73c6acf0d1 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411085e6e1853ba104019be4d18403ec02423_22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:13,596 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/ef2c8bb22d0e4516b4d7b56320df23ab, store: [table=TestAcidGuarantees family=A region=22b286596054a5e4265c6f73c6acf0d1] 2024-11-08T00:26:13,597 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/ef2c8bb22d0e4516b4d7b56320df23ab is 175, key is test_row_0/A:col10/1731025573133/Put/seqid=0 2024-11-08T00:26:13,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742427_1603 (size=30955) 2024-11-08T00:26:13,600 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/ef2c8bb22d0e4516b4d7b56320df23ab 2024-11-08T00:26:13,620 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/ccdc5e2b52904f77a06aab690f01dc0c is 50, key is test_row_0/B:col10/1731025573133/Put/seqid=0 2024-11-08T00:26:13,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742428_1604 (size=12001) 2024-11-08T00:26:13,626 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/ccdc5e2b52904f77a06aab690f01dc0c 2024-11-08T00:26:13,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/fbc50f51b31d414d88ee9fb57e269475 is 50, key is test_row_0/C:col10/1731025573133/Put/seqid=0 2024-11-08T00:26:13,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742429_1605 (size=12001) 2024-11-08T00:26:13,650 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/fbc50f51b31d414d88ee9fb57e269475 2024-11-08T00:26:13,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/ef2c8bb22d0e4516b4d7b56320df23ab as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/ef2c8bb22d0e4516b4d7b56320df23ab 2024-11-08T00:26:13,657 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/ef2c8bb22d0e4516b4d7b56320df23ab, entries=150, sequenceid=17, filesize=30.2 K 2024-11-08T00:26:13,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/ccdc5e2b52904f77a06aab690f01dc0c as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/ccdc5e2b52904f77a06aab690f01dc0c 2024-11-08T00:26:13,661 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/ccdc5e2b52904f77a06aab690f01dc0c, entries=150, sequenceid=17, filesize=11.7 K 2024-11-08T00:26:13,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/fbc50f51b31d414d88ee9fb57e269475 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/fbc50f51b31d414d88ee9fb57e269475 2024-11-08T00:26:13,664 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/fbc50f51b31d414d88ee9fb57e269475, entries=150, sequenceid=17, filesize=11.7 K 2024-11-08T00:26:13,665 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 22b286596054a5e4265c6f73c6acf0d1 in 532ms, sequenceid=17, compaction requested=false 2024-11-08T00:26:13,665 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-08T00:26:13,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 22b286596054a5e4265c6f73c6acf0d1: 2024-11-08T00:26:13,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-08T00:26:13,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:13,737 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 22b286596054a5e4265c6f73c6acf0d1 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-08T00:26:13,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411082b68ef4670374642b9fc5be0cf7a07d8_22b286596054a5e4265c6f73c6acf0d1 is 50, key is test_row_0/A:col10/1731025573157/Put/seqid=0 2024-11-08T00:26:13,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742430_1606 (size=12154) 2024-11-08T00:26:13,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-08T00:26:13,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:13,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. as already flushing 2024-11-08T00:26:13,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:13,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025633794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:13,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:13,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:13,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:13,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:13,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025633796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:13,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:13,801 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:13,801 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:13,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025633797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:13,801 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:13,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:13,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025633797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:13,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:13,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025633798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:13,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:13,802 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:13,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:13,802 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:13,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:13,802 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:13,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:13,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:13,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:13,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:13,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025633907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:13,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:13,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:13,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:13,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025633907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:13,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:13,910 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:13,910 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:13,910 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:13,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:13,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025633907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:13,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:13,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:13,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:13,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:13,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025633907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:13,911 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:13,911 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:13,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:13,911 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:13,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025633907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:13,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:13,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:13,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:14,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:14,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025634117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:14,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:14,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025634117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:14,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:14,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025634117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:14,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:14,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:14,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:14,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025634117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:14,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:14,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:14,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025634117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:14,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:14,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:14,120 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:14,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:14,120 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:14,120 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:14,120 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:14,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:14,120 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:14,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:14,120 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:14,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:14,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:14,149 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411082b68ef4670374642b9fc5be0cf7a07d8_22b286596054a5e4265c6f73c6acf0d1 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411082b68ef4670374642b9fc5be0cf7a07d8_22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:14,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/a283c9870e694513904e0905fcdeac13, store: [table=TestAcidGuarantees family=A region=22b286596054a5e4265c6f73c6acf0d1] 2024-11-08T00:26:14,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/a283c9870e694513904e0905fcdeac13 is 175, key is test_row_0/A:col10/1731025573157/Put/seqid=0 2024-11-08T00:26:14,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742431_1607 (size=30955) 2024-11-08T00:26:14,178 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/a283c9870e694513904e0905fcdeac13 2024-11-08T00:26:14,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/a05055c84b88406c8ad1aa4d613d8b0d is 50, key is test_row_0/B:col10/1731025573157/Put/seqid=0 2024-11-08T00:26:14,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742432_1608 (size=12001) 2024-11-08T00:26:14,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-08T00:26:14,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:14,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025634427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:14,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:14,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:14,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:14,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:14,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025634427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:14,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:14,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:14,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:14,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:14,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025634427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:14,430 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:14,430 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:14,430 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:14,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:14,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025634427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:14,431 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:14,431 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:14,431 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:14,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:14,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025634427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:14,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:14,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:14,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:14,587 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/a05055c84b88406c8ad1aa4d613d8b0d 2024-11-08T00:26:14,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/806985b1f8b84264847d5ff29373995e is 50, key is test_row_0/C:col10/1731025573157/Put/seqid=0 2024-11-08T00:26:14,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742433_1609 (size=12001) 2024-11-08T00:26:14,598 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/806985b1f8b84264847d5ff29373995e 2024-11-08T00:26:14,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/a283c9870e694513904e0905fcdeac13 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/a283c9870e694513904e0905fcdeac13 2024-11-08T00:26:14,606 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/a283c9870e694513904e0905fcdeac13, entries=150, sequenceid=40, filesize=30.2 K 2024-11-08T00:26:14,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/a05055c84b88406c8ad1aa4d613d8b0d as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/a05055c84b88406c8ad1aa4d613d8b0d 2024-11-08T00:26:14,609 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/a05055c84b88406c8ad1aa4d613d8b0d, entries=150, sequenceid=40, filesize=11.7 K 2024-11-08T00:26:14,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/806985b1f8b84264847d5ff29373995e as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/806985b1f8b84264847d5ff29373995e 2024-11-08T00:26:14,613 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/806985b1f8b84264847d5ff29373995e, entries=150, sequenceid=40, filesize=11.7 K 2024-11-08T00:26:14,614 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 22b286596054a5e4265c6f73c6acf0d1 in 877ms, sequenceid=40, compaction requested=false 2024-11-08T00:26:14,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 22b286596054a5e4265c6f73c6acf0d1: 2024-11-08T00:26:14,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:14,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-08T00:26:14,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-08T00:26:14,617 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-08T00:26:14,617 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4920 sec 2024-11-08T00:26:14,619 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.4980 sec 2024-11-08T00:26:14,715 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T00:26:14,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:14,940 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:26:14,940 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 22b286596054a5e4265c6f73c6acf0d1 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-08T00:26:14,945 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108604b1612b67946a3827f0d129ddb9578_22b286596054a5e4265c6f73c6acf0d1 is 50, key is test_row_0/A:col10/1731025573797/Put/seqid=0 2024-11-08T00:26:14,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742434_1610 (size=12154) 2024-11-08T00:26:14,953 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:14,957 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108604b1612b67946a3827f0d129ddb9578_22b286596054a5e4265c6f73c6acf0d1 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108604b1612b67946a3827f0d129ddb9578_22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:14,957 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/232afd805d87439ea774f15fe285d6af, store: [table=TestAcidGuarantees family=A region=22b286596054a5e4265c6f73c6acf0d1] 2024-11-08T00:26:14,958 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/232afd805d87439ea774f15fe285d6af is 175, key is test_row_0/A:col10/1731025573797/Put/seqid=0 2024-11-08T00:26:14,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:14,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025634956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:14,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742435_1611 (size=30955) 2024-11-08T00:26:14,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:14,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:14,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:14,963 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=56, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/232afd805d87439ea774f15fe285d6af 2024-11-08T00:26:14,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:14,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025634961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:14,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:14,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025634961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:14,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:14,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:14,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:14,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025634962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:14,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:14,967 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:14,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:14,967 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:14,967 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:14,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:14,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025634963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:14,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:14,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:14,967 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:14,967 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:14,967 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:14,982 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/70124e565243402ab7f0554adf229e8e is 50, key is test_row_0/B:col10/1731025573797/Put/seqid=0 2024-11-08T00:26:14,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742436_1612 (size=12001) 2024-11-08T00:26:15,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:15,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025635067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:15,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:15,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:15,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:15,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:15,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025635077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:15,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:15,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:15,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:15,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025635077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:15,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:15,079 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:15,079 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:15,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:15,080 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:15,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025635078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:15,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:15,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:15,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:15,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025635078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:15,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:15,080 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:15,080 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:15,080 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:15,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-08T00:26:15,267 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:26:15,268 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-08T00:26:15,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-08T00:26:15,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-08T00:26:15,269 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T00:26:15,269 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T00:26:15,269 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T00:26:15,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:15,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025635277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:15,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:15,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:15,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:15,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:15,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025635286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:15,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:15,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:15,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:15,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025635287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:15,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:15,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:15,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:15,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:15,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025635287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:15,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:15,291 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:15,291 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:15,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:15,291 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:15,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025635287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:15,291 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:15,291 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:15,291 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:15,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-08T00:26:15,390 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/70124e565243402ab7f0554adf229e8e 2024-11-08T00:26:15,396 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/a2a03013553b4b439489d4ccdbb24efa is 50, key is test_row_0/C:col10/1731025573797/Put/seqid=0 2024-11-08T00:26:15,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742437_1613 (size=12001) 2024-11-08T00:26:15,421 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-08T00:26:15,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:15,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. as already flushing 2024-11-08T00:26:15,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:15,421 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:15,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:15,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:15,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-08T00:26:15,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:15,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. as already flushing 2024-11-08T00:26:15,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:15,574 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:15,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:15,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:15,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-08T00:26:15,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:15,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025635587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:15,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:15,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:15,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:15,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:15,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025635597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:15,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:15,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:15,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:15,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:15,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025635597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:15,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:15,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:15,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:15,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:15,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025635598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:15,601 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:15,601 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:15,601 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:15,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:15,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025635598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:15,602 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:15,602 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:15,602 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:15,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-08T00:26:15,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:15,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. as already flushing 2024-11-08T00:26:15,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:15,726 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:15,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:15,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:15,801 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/a2a03013553b4b439489d4ccdbb24efa 2024-11-08T00:26:15,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/232afd805d87439ea774f15fe285d6af as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/232afd805d87439ea774f15fe285d6af 2024-11-08T00:26:15,807 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/232afd805d87439ea774f15fe285d6af, entries=150, sequenceid=56, filesize=30.2 K 2024-11-08T00:26:15,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/70124e565243402ab7f0554adf229e8e as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/70124e565243402ab7f0554adf229e8e 2024-11-08T00:26:15,810 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/70124e565243402ab7f0554adf229e8e, entries=150, sequenceid=56, filesize=11.7 K 2024-11-08T00:26:15,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/a2a03013553b4b439489d4ccdbb24efa as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/a2a03013553b4b439489d4ccdbb24efa 2024-11-08T00:26:15,813 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/a2a03013553b4b439489d4ccdbb24efa, entries=150, sequenceid=56, filesize=11.7 K 2024-11-08T00:26:15,813 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 22b286596054a5e4265c6f73c6acf0d1 in 873ms, sequenceid=56, compaction requested=true 2024-11-08T00:26:15,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 22b286596054a5e4265c6f73c6acf0d1: 2024-11-08T00:26:15,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22b286596054a5e4265c6f73c6acf0d1:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T00:26:15,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:26:15,814 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:26:15,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22b286596054a5e4265c6f73c6acf0d1:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T00:26:15,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:26:15,814 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:26:15,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22b286596054a5e4265c6f73c6acf0d1:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:26:15,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:26:15,815 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:26:15,815 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:26:15,815 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): 22b286596054a5e4265c6f73c6acf0d1/A is initiating minor compaction (all files) 2024-11-08T00:26:15,815 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): 22b286596054a5e4265c6f73c6acf0d1/B is initiating minor compaction (all files) 2024-11-08T00:26:15,815 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22b286596054a5e4265c6f73c6acf0d1/B in TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:15,815 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22b286596054a5e4265c6f73c6acf0d1/A in TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:15,815 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/ef2c8bb22d0e4516b4d7b56320df23ab, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/a283c9870e694513904e0905fcdeac13, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/232afd805d87439ea774f15fe285d6af] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp, totalSize=90.7 K 2024-11-08T00:26:15,815 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/ccdc5e2b52904f77a06aab690f01dc0c, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/a05055c84b88406c8ad1aa4d613d8b0d, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/70124e565243402ab7f0554adf229e8e] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp, totalSize=35.2 K 2024-11-08T00:26:15,815 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:15,815 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. files: [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/ef2c8bb22d0e4516b4d7b56320df23ab, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/a283c9870e694513904e0905fcdeac13, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/232afd805d87439ea774f15fe285d6af] 2024-11-08T00:26:15,815 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting ccdc5e2b52904f77a06aab690f01dc0c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731025573128 2024-11-08T00:26:15,815 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef2c8bb22d0e4516b4d7b56320df23ab, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731025573128 2024-11-08T00:26:15,815 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting a05055c84b88406c8ad1aa4d613d8b0d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731025573155 2024-11-08T00:26:15,815 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting a283c9870e694513904e0905fcdeac13, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731025573155 2024-11-08T00:26:15,816 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 70124e565243402ab7f0554adf229e8e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731025573794 2024-11-08T00:26:15,816 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 232afd805d87439ea774f15fe285d6af, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731025573794 2024-11-08T00:26:15,819 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=22b286596054a5e4265c6f73c6acf0d1] 2024-11-08T00:26:15,820 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22b286596054a5e4265c6f73c6acf0d1#B#compaction#520 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:26:15,821 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/f489aa1fd69e48ba82a900ec5709026e is 50, key is test_row_0/B:col10/1731025573797/Put/seqid=0 2024-11-08T00:26:15,823 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241108c6d57d484c0f407bbc3b22dd3744245b_22b286596054a5e4265c6f73c6acf0d1 store=[table=TestAcidGuarantees family=A region=22b286596054a5e4265c6f73c6acf0d1] 2024-11-08T00:26:15,824 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241108c6d57d484c0f407bbc3b22dd3744245b_22b286596054a5e4265c6f73c6acf0d1, store=[table=TestAcidGuarantees family=A region=22b286596054a5e4265c6f73c6acf0d1] 2024-11-08T00:26:15,824 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108c6d57d484c0f407bbc3b22dd3744245b_22b286596054a5e4265c6f73c6acf0d1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=22b286596054a5e4265c6f73c6acf0d1] 2024-11-08T00:26:15,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742438_1614 (size=12104) 2024-11-08T00:26:15,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742439_1615 (size=4469) 2024-11-08T00:26:15,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-08T00:26:15,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:15,879 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 22b286596054a5e4265c6f73c6acf0d1 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-08T00:26:15,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411081a005640951a4e6689617bf6b555dc2d_22b286596054a5e4265c6f73c6acf0d1 is 50, key is test_row_0/A:col10/1731025574960/Put/seqid=0 2024-11-08T00:26:15,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742440_1616 (size=12154) 2024-11-08T00:26:15,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-08T00:26:16,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:16,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. as already flushing 2024-11-08T00:26:16,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:16,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025636113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:16,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:16,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:16,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:16,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:16,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:16,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025636114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:16,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025636115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:16,119 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:16,119 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:16,119 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:16,119 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:16,119 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:16,119 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:16,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:16,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025636116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:16,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:16,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:16,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:16,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:16,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025636117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:16,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:16,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:16,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:16,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:16,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025636227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:16,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:16,228 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:16,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025636227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:16,229 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:16,229 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:16,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:16,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025636227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:16,229 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:16,229 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:16,229 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:16,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:16,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:16,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:16,230 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/f489aa1fd69e48ba82a900ec5709026e as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/f489aa1fd69e48ba82a900ec5709026e 2024-11-08T00:26:16,230 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22b286596054a5e4265c6f73c6acf0d1#A#compaction#521 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:26:16,230 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/a5f6c7f792a046489eeb2fe6708e2a44 is 175, key is test_row_0/A:col10/1731025573797/Put/seqid=0 2024-11-08T00:26:16,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:16,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025636228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:16,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:16,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025636228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:16,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:16,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:16,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:16,233 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22b286596054a5e4265c6f73c6acf0d1/B of 22b286596054a5e4265c6f73c6acf0d1 into f489aa1fd69e48ba82a900ec5709026e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:26:16,233 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22b286596054a5e4265c6f73c6acf0d1: 2024-11-08T00:26:16,233 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., storeName=22b286596054a5e4265c6f73c6acf0d1/B, priority=13, startTime=1731025575814; duration=0sec 2024-11-08T00:26:16,233 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:26:16,233 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22b286596054a5e4265c6f73c6acf0d1:B 2024-11-08T00:26:16,233 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:16,234 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:16,234 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:26:16,234 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:16,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742441_1617 (size=31058) 2024-11-08T00:26:16,235 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:26:16,235 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): 22b286596054a5e4265c6f73c6acf0d1/C is initiating minor compaction (all files) 2024-11-08T00:26:16,235 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22b286596054a5e4265c6f73c6acf0d1/C in TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:16,235 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/fbc50f51b31d414d88ee9fb57e269475, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/806985b1f8b84264847d5ff29373995e, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/a2a03013553b4b439489d4ccdbb24efa] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp, totalSize=35.2 K 2024-11-08T00:26:16,236 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting fbc50f51b31d414d88ee9fb57e269475, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731025573128 2024-11-08T00:26:16,237 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 806985b1f8b84264847d5ff29373995e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731025573155 2024-11-08T00:26:16,237 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting a2a03013553b4b439489d4ccdbb24efa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731025573794 2024-11-08T00:26:16,240 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/a5f6c7f792a046489eeb2fe6708e2a44 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/a5f6c7f792a046489eeb2fe6708e2a44 2024-11-08T00:26:16,244 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22b286596054a5e4265c6f73c6acf0d1/A of 22b286596054a5e4265c6f73c6acf0d1 into a5f6c7f792a046489eeb2fe6708e2a44(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:26:16,244 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22b286596054a5e4265c6f73c6acf0d1: 2024-11-08T00:26:16,244 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., storeName=22b286596054a5e4265c6f73c6acf0d1/A, priority=13, startTime=1731025575813; duration=0sec 2024-11-08T00:26:16,244 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:26:16,244 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22b286596054a5e4265c6f73c6acf0d1:A 2024-11-08T00:26:16,245 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22b286596054a5e4265c6f73c6acf0d1#C#compaction#523 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:26:16,245 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/ad8a4112bbab49e4a9dff46ea980ffd7 is 50, key is test_row_0/C:col10/1731025573797/Put/seqid=0 2024-11-08T00:26:16,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742442_1618 (size=12104) 2024-11-08T00:26:16,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:16,293 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411081a005640951a4e6689617bf6b555dc2d_22b286596054a5e4265c6f73c6acf0d1 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411081a005640951a4e6689617bf6b555dc2d_22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:16,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/34a8ed2db6c64f5189c03977d5e79b10, store: [table=TestAcidGuarantees family=A region=22b286596054a5e4265c6f73c6acf0d1] 2024-11-08T00:26:16,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/34a8ed2db6c64f5189c03977d5e79b10 is 175, key is test_row_0/A:col10/1731025574960/Put/seqid=0 2024-11-08T00:26:16,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742443_1619 (size=30955) 2024-11-08T00:26:16,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-08T00:26:16,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:16,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025636437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:16,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:16,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025636437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:16,440 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:16,440 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:16,440 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:16,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:16,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:16,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025636437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:16,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:16,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:16,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:16,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:16,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025636437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:16,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025636437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:16,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:16,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:16,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:16,440 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:16,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:16,440 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:16,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:16,440 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:16,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:16,652 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/ad8a4112bbab49e4a9dff46ea980ffd7 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/ad8a4112bbab49e4a9dff46ea980ffd7 2024-11-08T00:26:16,656 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22b286596054a5e4265c6f73c6acf0d1/C of 22b286596054a5e4265c6f73c6acf0d1 into ad8a4112bbab49e4a9dff46ea980ffd7(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:26:16,656 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22b286596054a5e4265c6f73c6acf0d1: 2024-11-08T00:26:16,656 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., storeName=22b286596054a5e4265c6f73c6acf0d1/C, priority=13, startTime=1731025575814; duration=0sec 2024-11-08T00:26:16,656 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:26:16,656 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22b286596054a5e4265c6f73c6acf0d1:C 2024-11-08T00:26:16,697 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=76, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/34a8ed2db6c64f5189c03977d5e79b10 2024-11-08T00:26:16,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/8463d10a3f5240c2bb6f1cd3f74d75a1 is 50, key is test_row_0/B:col10/1731025574960/Put/seqid=0 2024-11-08T00:26:16,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742444_1620 (size=12001) 2024-11-08T00:26:16,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:16,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025636747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:16,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:16,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025636747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:16,749 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:16,749 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:16,749 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:16,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:16,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:16,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025636748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:16,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:16,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:16,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:16,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:16,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:16,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:16,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025636748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:16,750 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:16,750 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:16,750 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:16,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:16,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025636748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:16,750 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:16,750 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:16,750 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:17,107 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/8463d10a3f5240c2bb6f1cd3f74d75a1 2024-11-08T00:26:17,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/9bf9df4a42ee4da5972d6b83c4da5f1a is 50, key is test_row_0/C:col10/1731025574960/Put/seqid=0 2024-11-08T00:26:17,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742445_1621 (size=12001) 2024-11-08T00:26:17,116 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/9bf9df4a42ee4da5972d6b83c4da5f1a 2024-11-08T00:26:17,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/34a8ed2db6c64f5189c03977d5e79b10 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/34a8ed2db6c64f5189c03977d5e79b10 2024-11-08T00:26:17,121 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/34a8ed2db6c64f5189c03977d5e79b10, entries=150, sequenceid=76, filesize=30.2 K 2024-11-08T00:26:17,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/8463d10a3f5240c2bb6f1cd3f74d75a1 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/8463d10a3f5240c2bb6f1cd3f74d75a1 2024-11-08T00:26:17,124 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/8463d10a3f5240c2bb6f1cd3f74d75a1, entries=150, sequenceid=76, filesize=11.7 K 2024-11-08T00:26:17,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/9bf9df4a42ee4da5972d6b83c4da5f1a as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/9bf9df4a42ee4da5972d6b83c4da5f1a 2024-11-08T00:26:17,132 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/9bf9df4a42ee4da5972d6b83c4da5f1a, entries=150, sequenceid=76, filesize=11.7 K 2024-11-08T00:26:17,133 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 22b286596054a5e4265c6f73c6acf0d1 in 1254ms, sequenceid=76, compaction requested=false 2024-11-08T00:26:17,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 22b286596054a5e4265c6f73c6acf0d1: 2024-11-08T00:26:17,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:17,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-08T00:26:17,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-08T00:26:17,136 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-08T00:26:17,136 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8650 sec 2024-11-08T00:26:17,137 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.8680 sec 2024-11-08T00:26:17,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:17,259 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:26:17,259 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 22b286596054a5e4265c6f73c6acf0d1 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-08T00:26:17,265 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411080237ef4db6de42ca97927ccb13e7dd0b_22b286596054a5e4265c6f73c6acf0d1 is 50, key is test_row_0/A:col10/1731025577259/Put/seqid=0 2024-11-08T00:26:17,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742446_1622 (size=12154) 2024-11-08T00:26:17,274 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:17,277 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411080237ef4db6de42ca97927ccb13e7dd0b_22b286596054a5e4265c6f73c6acf0d1 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411080237ef4db6de42ca97927ccb13e7dd0b_22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:17,278 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/3209449326ae4d4296fc162069668e43, store: [table=TestAcidGuarantees family=A region=22b286596054a5e4265c6f73c6acf0d1] 2024-11-08T00:26:17,278 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/3209449326ae4d4296fc162069668e43 is 175, key is test_row_0/A:col10/1731025577259/Put/seqid=0 2024-11-08T00:26:17,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:17,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025637274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:17,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:17,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:17,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:17,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:17,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025637275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:17,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:17,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025637275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:17,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:17,280 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:17,281 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:17,281 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:17,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025637276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:17,281 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:17,281 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:17,281 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:17,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:17,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:17,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025637276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:17,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:17,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:17,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:17,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:17,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:17,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742447_1623 (size=30955) 2024-11-08T00:26:17,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:17,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025637387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:17,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:17,389 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:17,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025637387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:17,389 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:17,389 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:17,389 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:17,389 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:17,389 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:17,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:17,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025637387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:17,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:17,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:17,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:17,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:17,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025637387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:17,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:17,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:17,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:17,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:17,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025637388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:17,391 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:17,391 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:17,391 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:17,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-08T00:26:17,417 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:26:17,418 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-08T00:26:17,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-08T00:26:17,419 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T00:26:17,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-08T00:26:17,419 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T00:26:17,420 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T00:26:17,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-08T00:26:17,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-08T00:26:17,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:17,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. as already flushing 2024-11-08T00:26:17,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:17,572 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:17,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:17,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:17,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:17,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025637597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:17,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:17,599 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:17,599 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:17,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025637597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:17,599 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:17,600 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:17,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:17,600 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:17,600 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:17,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025637598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:17,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:17,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:17,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:17,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:17,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025637598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:17,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:17,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:17,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:17,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:17,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025637598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:17,600 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:17,600 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:17,601 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:17,682 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=97, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/3209449326ae4d4296fc162069668e43 2024-11-08T00:26:17,688 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/6fd7cbfede4a4f729edbe9e7a8e3dde0 is 50, key is test_row_0/B:col10/1731025577259/Put/seqid=0 2024-11-08T00:26:17,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742448_1624 (size=12001) 2024-11-08T00:26:17,724 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-08T00:26:17,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:17,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. as already flushing 2024-11-08T00:26:17,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:17,725 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:17,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:17,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:17,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-08T00:26:17,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-08T00:26:17,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:17,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. as already flushing 2024-11-08T00:26:17,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:17,877 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:17,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:17,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:17,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:17,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025637907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:17,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:17,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:17,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025637907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:17,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:17,909 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:17,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:17,910 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:17,910 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:17,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025637907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:17,910 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:17,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:17,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025637907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:17,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:17,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:17,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:17,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:17,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:17,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:17,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:17,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025637907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:17,910 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:17,910 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:17,910 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:18,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-08T00:26:18,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:18,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. as already flushing 2024-11-08T00:26:18,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:18,030 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:18,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:18,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:18,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-08T00:26:18,109 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/6fd7cbfede4a4f729edbe9e7a8e3dde0 2024-11-08T00:26:18,116 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/653e3ef9faff4b14b1278b534a548963 is 50, key is test_row_0/C:col10/1731025577259/Put/seqid=0 2024-11-08T00:26:18,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742449_1625 (size=12001) 2024-11-08T00:26:18,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-08T00:26:18,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:18,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. as already flushing 2024-11-08T00:26:18,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:18,182 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:18,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:18,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:18,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-08T00:26:18,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:18,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. as already flushing 2024-11-08T00:26:18,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:18,335 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:18,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:18,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:18,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:18,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025638417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:18,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:18,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:18,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025638417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:18,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:18,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:18,419 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:18,419 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:18,419 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:18,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:18,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025638417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:18,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:18,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:18,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025638417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:18,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:18,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:18,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:18,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:18,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:18,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:18,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025638417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:18,420 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:18,420 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:18,420 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:18,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-08T00:26:18,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:18,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. as already flushing 2024-11-08T00:26:18,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:18,488 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:18,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:18,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:18,519 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/653e3ef9faff4b14b1278b534a548963 2024-11-08T00:26:18,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/3209449326ae4d4296fc162069668e43 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/3209449326ae4d4296fc162069668e43 2024-11-08T00:26:18,525 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/3209449326ae4d4296fc162069668e43, entries=150, sequenceid=97, filesize=30.2 K 2024-11-08T00:26:18,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/6fd7cbfede4a4f729edbe9e7a8e3dde0 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/6fd7cbfede4a4f729edbe9e7a8e3dde0 2024-11-08T00:26:18,528 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/6fd7cbfede4a4f729edbe9e7a8e3dde0, entries=150, sequenceid=97, filesize=11.7 K 2024-11-08T00:26:18,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/653e3ef9faff4b14b1278b534a548963 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/653e3ef9faff4b14b1278b534a548963 2024-11-08T00:26:18,532 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/653e3ef9faff4b14b1278b534a548963, entries=150, sequenceid=97, filesize=11.7 K 2024-11-08T00:26:18,532 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 22b286596054a5e4265c6f73c6acf0d1 in 1273ms, sequenceid=97, compaction requested=true 2024-11-08T00:26:18,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 22b286596054a5e4265c6f73c6acf0d1: 2024-11-08T00:26:18,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22b286596054a5e4265c6f73c6acf0d1:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T00:26:18,533 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:26:18,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:26:18,533 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:26:18,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22b286596054a5e4265c6f73c6acf0d1:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T00:26:18,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:26:18,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22b286596054a5e4265c6f73c6acf0d1:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:26:18,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:26:18,535 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:26:18,535 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:26:18,535 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): 22b286596054a5e4265c6f73c6acf0d1/B is initiating minor compaction (all files) 2024-11-08T00:26:18,535 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): 22b286596054a5e4265c6f73c6acf0d1/A is initiating minor compaction (all files) 2024-11-08T00:26:18,535 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22b286596054a5e4265c6f73c6acf0d1/B in TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:18,535 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22b286596054a5e4265c6f73c6acf0d1/A in TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:18,535 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/a5f6c7f792a046489eeb2fe6708e2a44, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/34a8ed2db6c64f5189c03977d5e79b10, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/3209449326ae4d4296fc162069668e43] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp, totalSize=90.8 K 2024-11-08T00:26:18,535 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/f489aa1fd69e48ba82a900ec5709026e, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/8463d10a3f5240c2bb6f1cd3f74d75a1, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/6fd7cbfede4a4f729edbe9e7a8e3dde0] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp, totalSize=35.3 K 2024-11-08T00:26:18,535 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:18,535 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. files: [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/a5f6c7f792a046489eeb2fe6708e2a44, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/34a8ed2db6c64f5189c03977d5e79b10, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/3209449326ae4d4296fc162069668e43] 2024-11-08T00:26:18,535 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5f6c7f792a046489eeb2fe6708e2a44, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731025573794 2024-11-08T00:26:18,535 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting f489aa1fd69e48ba82a900ec5709026e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731025573794 2024-11-08T00:26:18,536 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 8463d10a3f5240c2bb6f1cd3f74d75a1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731025574959 2024-11-08T00:26:18,536 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34a8ed2db6c64f5189c03977d5e79b10, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731025574959 2024-11-08T00:26:18,536 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3209449326ae4d4296fc162069668e43, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731025576110 2024-11-08T00:26:18,536 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 6fd7cbfede4a4f729edbe9e7a8e3dde0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731025576110 2024-11-08T00:26:18,554 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22b286596054a5e4265c6f73c6acf0d1#B#compaction#529 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:26:18,555 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/1e502dbf63ee4f74bb02901664c031c5 is 50, key is test_row_0/B:col10/1731025577259/Put/seqid=0 2024-11-08T00:26:18,556 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=22b286596054a5e4265c6f73c6acf0d1] 2024-11-08T00:26:18,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-08T00:26:18,563 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241108be080d13b0d14d1b9f696b9b16187135_22b286596054a5e4265c6f73c6acf0d1 store=[table=TestAcidGuarantees family=A region=22b286596054a5e4265c6f73c6acf0d1] 2024-11-08T00:26:18,565 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241108be080d13b0d14d1b9f696b9b16187135_22b286596054a5e4265c6f73c6acf0d1, store=[table=TestAcidGuarantees family=A region=22b286596054a5e4265c6f73c6acf0d1] 2024-11-08T00:26:18,565 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108be080d13b0d14d1b9f696b9b16187135_22b286596054a5e4265c6f73c6acf0d1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=22b286596054a5e4265c6f73c6acf0d1] 2024-11-08T00:26:18,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742450_1626 (size=12207) 2024-11-08T00:26:18,587 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/1e502dbf63ee4f74bb02901664c031c5 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/1e502dbf63ee4f74bb02901664c031c5 2024-11-08T00:26:18,592 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22b286596054a5e4265c6f73c6acf0d1/B of 22b286596054a5e4265c6f73c6acf0d1 into 1e502dbf63ee4f74bb02901664c031c5(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:26:18,592 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22b286596054a5e4265c6f73c6acf0d1: 2024-11-08T00:26:18,592 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., storeName=22b286596054a5e4265c6f73c6acf0d1/B, priority=13, startTime=1731025578533; duration=0sec 2024-11-08T00:26:18,592 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:26:18,592 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22b286596054a5e4265c6f73c6acf0d1:B 2024-11-08T00:26:18,592 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:26:18,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742451_1627 (size=4469) 2024-11-08T00:26:18,595 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:26:18,595 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): 22b286596054a5e4265c6f73c6acf0d1/C is initiating minor compaction (all files) 2024-11-08T00:26:18,595 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22b286596054a5e4265c6f73c6acf0d1/C in TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:18,595 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/ad8a4112bbab49e4a9dff46ea980ffd7, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/9bf9df4a42ee4da5972d6b83c4da5f1a, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/653e3ef9faff4b14b1278b534a548963] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp, totalSize=35.3 K 2024-11-08T00:26:18,595 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22b286596054a5e4265c6f73c6acf0d1#A#compaction#530 average throughput is 0.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:26:18,596 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/1ed3ace8f08142a59b691b167cfed180 is 175, key is test_row_0/A:col10/1731025577259/Put/seqid=0 2024-11-08T00:26:18,596 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting ad8a4112bbab49e4a9dff46ea980ffd7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731025573794 2024-11-08T00:26:18,597 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bf9df4a42ee4da5972d6b83c4da5f1a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731025574959 2024-11-08T00:26:18,597 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 653e3ef9faff4b14b1278b534a548963, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731025576110 2024-11-08T00:26:18,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742452_1628 (size=31161) 2024-11-08T00:26:18,630 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22b286596054a5e4265c6f73c6acf0d1#C#compaction#531 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:26:18,630 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/8039654462934ed886a240d10506afc5 is 50, key is test_row_0/C:col10/1731025577259/Put/seqid=0 2024-11-08T00:26:18,637 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/1ed3ace8f08142a59b691b167cfed180 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/1ed3ace8f08142a59b691b167cfed180 2024-11-08T00:26:18,640 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-08T00:26:18,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:18,640 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 22b286596054a5e4265c6f73c6acf0d1 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-08T00:26:18,644 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22b286596054a5e4265c6f73c6acf0d1/A of 22b286596054a5e4265c6f73c6acf0d1 into 1ed3ace8f08142a59b691b167cfed180(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:26:18,644 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22b286596054a5e4265c6f73c6acf0d1: 2024-11-08T00:26:18,644 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., storeName=22b286596054a5e4265c6f73c6acf0d1/A, priority=13, startTime=1731025578532; duration=0sec 2024-11-08T00:26:18,644 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:26:18,644 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22b286596054a5e4265c6f73c6acf0d1:A 2024-11-08T00:26:18,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742453_1629 (size=12207) 2024-11-08T00:26:18,689 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/8039654462934ed886a240d10506afc5 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/8039654462934ed886a240d10506afc5 2024-11-08T00:26:18,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411084bd9e2b8f170478690c3970b277f1a17_22b286596054a5e4265c6f73c6acf0d1 is 50, key is test_row_0/A:col10/1731025577274/Put/seqid=0 2024-11-08T00:26:18,698 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22b286596054a5e4265c6f73c6acf0d1/C of 22b286596054a5e4265c6f73c6acf0d1 into 8039654462934ed886a240d10506afc5(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:26:18,698 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22b286596054a5e4265c6f73c6acf0d1: 2024-11-08T00:26:18,698 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., storeName=22b286596054a5e4265c6f73c6acf0d1/C, priority=13, startTime=1731025578533; duration=0sec 2024-11-08T00:26:18,698 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:26:18,698 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22b286596054a5e4265c6f73c6acf0d1:C 2024-11-08T00:26:18,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742454_1630 (size=12154) 2024-11-08T00:26:19,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:19,131 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411084bd9e2b8f170478690c3970b277f1a17_22b286596054a5e4265c6f73c6acf0d1 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411084bd9e2b8f170478690c3970b277f1a17_22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:19,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/e7f12ca8eeab448c8201c7a3f14d6a4a, store: [table=TestAcidGuarantees family=A region=22b286596054a5e4265c6f73c6acf0d1] 2024-11-08T00:26:19,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/e7f12ca8eeab448c8201c7a3f14d6a4a is 175, key is test_row_0/A:col10/1731025577274/Put/seqid=0 2024-11-08T00:26:19,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742455_1631 (size=30955) 2024-11-08T00:26:19,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:19,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. as already flushing 2024-11-08T00:26:19,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:19,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025639448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:19,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:19,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:19,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:19,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:19,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025639449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:19,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:19,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:19,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:19,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:19,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025639450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:19,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:19,456 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:19,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025639451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:19,456 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:19,456 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:19,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:19,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025639452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:19,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:19,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:19,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:19,457 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:19,457 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:19,457 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:19,551 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=116, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/e7f12ca8eeab448c8201c7a3f14d6a4a 2024-11-08T00:26:19,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:19,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:19,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025639558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:19,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025639559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:19,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:19,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:19,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:19,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:19,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:19,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:19,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/8797895ce3d645b99b54dc7b6d8332fd is 50, key is test_row_0/B:col10/1731025577274/Put/seqid=0 2024-11-08T00:26:19,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-08T00:26:19,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:19,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025639567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:19,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:19,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:19,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:19,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025639567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:19,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:19,572 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:19,572 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:19,572 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:19,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:19,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025639568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:19,573 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:19,573 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:19,573 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:19,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742456_1632 (size=12001) 2024-11-08T00:26:19,603 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/8797895ce3d645b99b54dc7b6d8332fd 2024-11-08T00:26:19,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/180a2b66a3ab46318244a474bd3f6b56 is 50, key is test_row_0/C:col10/1731025577274/Put/seqid=0 2024-11-08T00:26:19,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742457_1633 (size=12001) 2024-11-08T00:26:19,613 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/180a2b66a3ab46318244a474bd3f6b56 2024-11-08T00:26:19,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/e7f12ca8eeab448c8201c7a3f14d6a4a as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/e7f12ca8eeab448c8201c7a3f14d6a4a 2024-11-08T00:26:19,621 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/e7f12ca8eeab448c8201c7a3f14d6a4a, entries=150, sequenceid=116, filesize=30.2 K 2024-11-08T00:26:19,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/8797895ce3d645b99b54dc7b6d8332fd as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/8797895ce3d645b99b54dc7b6d8332fd 2024-11-08T00:26:19,625 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/8797895ce3d645b99b54dc7b6d8332fd, entries=150, sequenceid=116, filesize=11.7 K 2024-11-08T00:26:19,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/180a2b66a3ab46318244a474bd3f6b56 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/180a2b66a3ab46318244a474bd3f6b56 2024-11-08T00:26:19,631 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/180a2b66a3ab46318244a474bd3f6b56, entries=150, sequenceid=116, filesize=11.7 K 2024-11-08T00:26:19,632 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 22b286596054a5e4265c6f73c6acf0d1 in 991ms, sequenceid=116, compaction requested=false 2024-11-08T00:26:19,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 22b286596054a5e4265c6f73c6acf0d1: 2024-11-08T00:26:19,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:19,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-08T00:26:19,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-08T00:26:19,634 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-08T00:26:19,635 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2130 sec 2024-11-08T00:26:19,636 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 2.2170 sec 2024-11-08T00:26:19,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:19,770 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:26:19,770 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 22b286596054a5e4265c6f73c6acf0d1 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-08T00:26:19,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411089d604e62cf804fabb2872d3ea806a776_22b286596054a5e4265c6f73c6acf0d1 is 50, key is test_row_0/A:col10/1731025579450/Put/seqid=0 2024-11-08T00:26:19,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:19,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025639794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:19,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:19,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:19,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:19,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:19,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025639796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:19,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:19,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025639797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:19,803 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:19,803 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:19,803 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:19,803 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:19,803 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:19,803 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:19,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:19,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025639800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:19,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:19,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:19,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:19,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:19,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025639803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:19,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:19,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:19,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:19,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742458_1634 (size=12304) 2024-11-08T00:26:19,827 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:19,831 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411089d604e62cf804fabb2872d3ea806a776_22b286596054a5e4265c6f73c6acf0d1 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411089d604e62cf804fabb2872d3ea806a776_22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:19,833 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/bbb35a1bcef647a3b87294f1a9db9cb1, store: [table=TestAcidGuarantees family=A region=22b286596054a5e4265c6f73c6acf0d1] 2024-11-08T00:26:19,834 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/bbb35a1bcef647a3b87294f1a9db9cb1 is 175, key is test_row_0/A:col10/1731025579450/Put/seqid=0 2024-11-08T00:26:19,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742459_1635 (size=31105) 2024-11-08T00:26:19,877 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=137, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/bbb35a1bcef647a3b87294f1a9db9cb1 2024-11-08T00:26:19,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/057b5fb0f7a1462488af01d61cc0b618 is 50, key is test_row_0/B:col10/1731025579450/Put/seqid=0 2024-11-08T00:26:19,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:19,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025639907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:19,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:19,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:19,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:19,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:19,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025639907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:19,912 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:19,912 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:19,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:19,912 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:19,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025639908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:19,913 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:19,913 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:19,913 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:19,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:19,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025639917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:19,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:19,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:19,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:19,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:19,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025639917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:19,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:19,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:19,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:19,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742460_1636 (size=12151) 2024-11-08T00:26:19,958 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/057b5fb0f7a1462488af01d61cc0b618 2024-11-08T00:26:19,970 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/3fcd612df23540c59b0063051f80248f is 50, key is test_row_0/C:col10/1731025579450/Put/seqid=0 2024-11-08T00:26:19,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742461_1637 (size=12151) 2024-11-08T00:26:19,987 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/3fcd612df23540c59b0063051f80248f 2024-11-08T00:26:19,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/bbb35a1bcef647a3b87294f1a9db9cb1 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/bbb35a1bcef647a3b87294f1a9db9cb1 2024-11-08T00:26:19,999 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/bbb35a1bcef647a3b87294f1a9db9cb1, entries=150, sequenceid=137, filesize=30.4 K 2024-11-08T00:26:20,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/057b5fb0f7a1462488af01d61cc0b618 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/057b5fb0f7a1462488af01d61cc0b618 2024-11-08T00:26:20,012 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/057b5fb0f7a1462488af01d61cc0b618, entries=150, sequenceid=137, filesize=11.9 K 2024-11-08T00:26:20,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/3fcd612df23540c59b0063051f80248f as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/3fcd612df23540c59b0063051f80248f 2024-11-08T00:26:20,020 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/3fcd612df23540c59b0063051f80248f, entries=150, sequenceid=137, filesize=11.9 K 2024-11-08T00:26:20,022 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for 22b286596054a5e4265c6f73c6acf0d1 in 251ms, sequenceid=137, compaction requested=true 2024-11-08T00:26:20,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 22b286596054a5e4265c6f73c6acf0d1: 2024-11-08T00:26:20,022 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:26:20,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22b286596054a5e4265c6f73c6acf0d1:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T00:26:20,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:26:20,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22b286596054a5e4265c6f73c6acf0d1:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T00:26:20,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:26:20,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22b286596054a5e4265c6f73c6acf0d1:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:26:20,022 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:26:20,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:26:20,023 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93221 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:26:20,023 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): 22b286596054a5e4265c6f73c6acf0d1/A is initiating minor compaction (all files) 2024-11-08T00:26:20,023 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22b286596054a5e4265c6f73c6acf0d1/A in TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:20,023 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:26:20,023 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/1ed3ace8f08142a59b691b167cfed180, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/e7f12ca8eeab448c8201c7a3f14d6a4a, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/bbb35a1bcef647a3b87294f1a9db9cb1] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp, totalSize=91.0 K 2024-11-08T00:26:20,023 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): 22b286596054a5e4265c6f73c6acf0d1/B is initiating minor compaction (all files) 2024-11-08T00:26:20,023 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:20,023 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22b286596054a5e4265c6f73c6acf0d1/B in TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:20,023 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. files: [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/1ed3ace8f08142a59b691b167cfed180, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/e7f12ca8eeab448c8201c7a3f14d6a4a, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/bbb35a1bcef647a3b87294f1a9db9cb1] 2024-11-08T00:26:20,023 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/1e502dbf63ee4f74bb02901664c031c5, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/8797895ce3d645b99b54dc7b6d8332fd, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/057b5fb0f7a1462488af01d61cc0b618] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp, totalSize=35.5 K 2024-11-08T00:26:20,024 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ed3ace8f08142a59b691b167cfed180, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731025576110 2024-11-08T00:26:20,024 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e502dbf63ee4f74bb02901664c031c5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731025576110 2024-11-08T00:26:20,024 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7f12ca8eeab448c8201c7a3f14d6a4a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731025577273 2024-11-08T00:26:20,024 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 8797895ce3d645b99b54dc7b6d8332fd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731025577273 2024-11-08T00:26:20,024 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 057b5fb0f7a1462488af01d61cc0b618, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731025579448 2024-11-08T00:26:20,024 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbb35a1bcef647a3b87294f1a9db9cb1, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731025579448 2024-11-08T00:26:20,042 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22b286596054a5e4265c6f73c6acf0d1#B#compaction#538 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:26:20,042 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/13b2b7cc0dc44fcaa7c435ac4237597a is 50, key is test_row_0/B:col10/1731025579450/Put/seqid=0 2024-11-08T00:26:20,049 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=22b286596054a5e4265c6f73c6acf0d1] 2024-11-08T00:26:20,065 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411083102aab4a2c143029b1ed50c5bc93f22_22b286596054a5e4265c6f73c6acf0d1 store=[table=TestAcidGuarantees family=A region=22b286596054a5e4265c6f73c6acf0d1] 2024-11-08T00:26:20,067 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411083102aab4a2c143029b1ed50c5bc93f22_22b286596054a5e4265c6f73c6acf0d1, store=[table=TestAcidGuarantees family=A region=22b286596054a5e4265c6f73c6acf0d1] 2024-11-08T00:26:20,067 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411083102aab4a2c143029b1ed50c5bc93f22_22b286596054a5e4265c6f73c6acf0d1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=22b286596054a5e4265c6f73c6acf0d1] 2024-11-08T00:26:20,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742462_1638 (size=12459) 2024-11-08T00:26:20,100 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/13b2b7cc0dc44fcaa7c435ac4237597a as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/13b2b7cc0dc44fcaa7c435ac4237597a 2024-11-08T00:26:20,104 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22b286596054a5e4265c6f73c6acf0d1/B of 22b286596054a5e4265c6f73c6acf0d1 into 13b2b7cc0dc44fcaa7c435ac4237597a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:26:20,104 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22b286596054a5e4265c6f73c6acf0d1: 2024-11-08T00:26:20,104 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., storeName=22b286596054a5e4265c6f73c6acf0d1/B, priority=13, startTime=1731025580022; duration=0sec 2024-11-08T00:26:20,104 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:26:20,104 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22b286596054a5e4265c6f73c6acf0d1:B 2024-11-08T00:26:20,104 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:26:20,105 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:26:20,105 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): 22b286596054a5e4265c6f73c6acf0d1/C is initiating minor compaction (all files) 2024-11-08T00:26:20,105 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22b286596054a5e4265c6f73c6acf0d1/C in TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:20,105 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/8039654462934ed886a240d10506afc5, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/180a2b66a3ab46318244a474bd3f6b56, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/3fcd612df23540c59b0063051f80248f] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp, totalSize=35.5 K 2024-11-08T00:26:20,106 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 8039654462934ed886a240d10506afc5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731025576110 2024-11-08T00:26:20,106 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 180a2b66a3ab46318244a474bd3f6b56, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731025577273 2024-11-08T00:26:20,106 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fcd612df23540c59b0063051f80248f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731025579448 2024-11-08T00:26:20,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:20,121 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:26:20,121 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 22b286596054a5e4265c6f73c6acf0d1 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-08T00:26:20,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742463_1639 (size=4469) 2024-11-08T00:26:20,132 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22b286596054a5e4265c6f73c6acf0d1#C#compaction#540 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:26:20,132 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/3c07df97f0c74d0fb599a5685a6877b9 is 50, key is test_row_0/C:col10/1731025579450/Put/seqid=0 2024-11-08T00:26:20,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:20,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025640141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:20,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:20,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:20,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:20,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:20,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025640142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:20,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:20,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025640143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:20,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:20,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:20,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:20,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:20,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:20,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:20,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:20,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025640147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:20,156 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:20,156 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:20,156 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:20,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:20,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025640152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:20,161 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:20,161 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:20,161 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:20,175 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108d8b3bde183904f828e37633ceba247a6_22b286596054a5e4265c6f73c6acf0d1 is 50, key is test_row_0/A:col10/1731025579797/Put/seqid=0 2024-11-08T00:26:20,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742464_1640 (size=12459) 2024-11-08T00:26:20,210 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/3c07df97f0c74d0fb599a5685a6877b9 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/3c07df97f0c74d0fb599a5685a6877b9 2024-11-08T00:26:20,216 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22b286596054a5e4265c6f73c6acf0d1/C of 22b286596054a5e4265c6f73c6acf0d1 into 3c07df97f0c74d0fb599a5685a6877b9(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:26:20,216 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22b286596054a5e4265c6f73c6acf0d1: 2024-11-08T00:26:20,216 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., storeName=22b286596054a5e4265c6f73c6acf0d1/C, priority=13, startTime=1731025580022; duration=0sec 2024-11-08T00:26:20,216 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:26:20,216 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22b286596054a5e4265c6f73c6acf0d1:C 2024-11-08T00:26:20,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742465_1641 (size=14794) 2024-11-08T00:26:20,218 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:20,222 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108d8b3bde183904f828e37633ceba247a6_22b286596054a5e4265c6f73c6acf0d1 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108d8b3bde183904f828e37633ceba247a6_22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:20,223 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/92704af1b09344a6bd680916e527922e, store: [table=TestAcidGuarantees family=A region=22b286596054a5e4265c6f73c6acf0d1] 2024-11-08T00:26:20,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/92704af1b09344a6bd680916e527922e is 175, key is test_row_0/A:col10/1731025579797/Put/seqid=0 2024-11-08T00:26:20,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:20,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025640258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:20,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:20,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:20,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:20,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:20,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:20,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025640258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:20,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025640258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:20,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:20,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:20,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:20,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742466_1642 (size=39749) 2024-11-08T00:26:20,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:20,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:20,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:20,262 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=158, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/92704af1b09344a6bd680916e527922e 2024-11-08T00:26:20,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:20,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025640267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:20,269 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:20,269 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:20,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:20,269 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:20,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025640268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:20,269 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:20,269 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:20,270 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:20,278 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/74e17db025a74094ac3bcc0c1c0f3b1c is 50, key is test_row_0/B:col10/1731025579797/Put/seqid=0 2024-11-08T00:26:20,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742467_1643 (size=12151) 2024-11-08T00:26:20,315 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/74e17db025a74094ac3bcc0c1c0f3b1c 2024-11-08T00:26:20,344 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/8b00792d0c0d4f16bad5dd9becf4e428 is 50, key is test_row_0/C:col10/1731025579797/Put/seqid=0 2024-11-08T00:26:20,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742468_1644 (size=12151) 2024-11-08T00:26:20,389 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/8b00792d0c0d4f16bad5dd9becf4e428 2024-11-08T00:26:20,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/92704af1b09344a6bd680916e527922e as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/92704af1b09344a6bd680916e527922e 2024-11-08T00:26:20,403 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/92704af1b09344a6bd680916e527922e, entries=200, sequenceid=158, filesize=38.8 K 2024-11-08T00:26:20,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/74e17db025a74094ac3bcc0c1c0f3b1c as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/74e17db025a74094ac3bcc0c1c0f3b1c 2024-11-08T00:26:20,411 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/74e17db025a74094ac3bcc0c1c0f3b1c, entries=150, sequenceid=158, filesize=11.9 K 2024-11-08T00:26:20,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/8b00792d0c0d4f16bad5dd9becf4e428 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/8b00792d0c0d4f16bad5dd9becf4e428 2024-11-08T00:26:20,417 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/8b00792d0c0d4f16bad5dd9becf4e428, entries=150, sequenceid=158, filesize=11.9 K 2024-11-08T00:26:20,417 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 22b286596054a5e4265c6f73c6acf0d1 in 296ms, sequenceid=158, compaction requested=false 2024-11-08T00:26:20,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 22b286596054a5e4265c6f73c6acf0d1: 2024-11-08T00:26:20,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:20,471 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:26:20,471 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 22b286596054a5e4265c6f73c6acf0d1 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-08T00:26:20,483 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108828fb5ef01394621854944824ba1b333_22b286596054a5e4265c6f73c6acf0d1 is 50, key is test_row_0/A:col10/1731025580469/Put/seqid=0 2024-11-08T00:26:20,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:20,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025640489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:20,498 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:20,498 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:20,498 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:20,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:20,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025640495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:20,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:20,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025640495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:20,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:20,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:20,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:20,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:20,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:20,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:20,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:20,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025640497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:20,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:20,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:20,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:20,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:20,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025640497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:20,505 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:20,505 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:20,505 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:20,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742469_1645 (size=14794) 2024-11-08T00:26:20,528 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22b286596054a5e4265c6f73c6acf0d1#A#compaction#539 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:26:20,529 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/b7541ca9eb774e10ba90a134449f8b13 is 175, key is test_row_0/A:col10/1731025579450/Put/seqid=0 2024-11-08T00:26:20,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742470_1646 (size=31413) 2024-11-08T00:26:20,556 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/b7541ca9eb774e10ba90a134449f8b13 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/b7541ca9eb774e10ba90a134449f8b13 2024-11-08T00:26:20,563 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22b286596054a5e4265c6f73c6acf0d1/A of 22b286596054a5e4265c6f73c6acf0d1 into b7541ca9eb774e10ba90a134449f8b13(size=30.7 K), total size for store is 69.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:26:20,563 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22b286596054a5e4265c6f73c6acf0d1: 2024-11-08T00:26:20,563 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., storeName=22b286596054a5e4265c6f73c6acf0d1/A, priority=13, startTime=1731025580022; duration=0sec 2024-11-08T00:26:20,564 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:26:20,564 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22b286596054a5e4265c6f73c6acf0d1:A 2024-11-08T00:26:20,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:20,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025640607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:20,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:20,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025640608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:20,611 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:20,611 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:20,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:20,611 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:20,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:20,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:20,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:20,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:20,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025640608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:20,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025640608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:20,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:20,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025640609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:20,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:20,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:20,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:20,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:20,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:20,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:20,617 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:20,617 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:20,617 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:20,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:20,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025640817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:20,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:20,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:20,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:20,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:20,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025640817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:20,821 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:20,821 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:20,821 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:20,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:20,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025640827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:20,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:20,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:20,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025640827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:20,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025640828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:20,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:20,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:20,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:20,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:20,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:20,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:20,832 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:20,832 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:20,832 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:20,922 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:20,926 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108828fb5ef01394621854944824ba1b333_22b286596054a5e4265c6f73c6acf0d1 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108828fb5ef01394621854944824ba1b333_22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:20,927 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/fa7eea9f75824ba7b5acff453eb5ab6b, store: [table=TestAcidGuarantees family=A region=22b286596054a5e4265c6f73c6acf0d1] 2024-11-08T00:26:20,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/fa7eea9f75824ba7b5acff453eb5ab6b is 175, key is test_row_0/A:col10/1731025580469/Put/seqid=0 2024-11-08T00:26:20,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742471_1647 (size=39749) 2024-11-08T00:26:20,943 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=178, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/fa7eea9f75824ba7b5acff453eb5ab6b 2024-11-08T00:26:20,951 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/091b7dae942f4398a2be115d3796aed7 is 50, key is test_row_0/B:col10/1731025580469/Put/seqid=0 2024-11-08T00:26:20,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742472_1648 (size=12151) 2024-11-08T00:26:21,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:21,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025641127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:21,130 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:21,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:21,130 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:21,130 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:21,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025641127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:21,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:21,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:21,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:21,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:21,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025641137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:21,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:21,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:21,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:21,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:21,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025641137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:21,140 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:21,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:21,140 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:21,141 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:21,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025641137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:21,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:21,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:21,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:21,370 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/091b7dae942f4398a2be115d3796aed7 2024-11-08T00:26:21,376 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/665538b71af64fc9a3d9ace3896b7a3a is 50, key is test_row_0/C:col10/1731025580469/Put/seqid=0 2024-11-08T00:26:21,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742473_1649 (size=12151) 2024-11-08T00:26:21,378 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/665538b71af64fc9a3d9ace3896b7a3a 2024-11-08T00:26:21,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/fa7eea9f75824ba7b5acff453eb5ab6b as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/fa7eea9f75824ba7b5acff453eb5ab6b 2024-11-08T00:26:21,384 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/fa7eea9f75824ba7b5acff453eb5ab6b, entries=200, sequenceid=178, filesize=38.8 K 2024-11-08T00:26:21,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/091b7dae942f4398a2be115d3796aed7 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/091b7dae942f4398a2be115d3796aed7 2024-11-08T00:26:21,387 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/091b7dae942f4398a2be115d3796aed7, entries=150, sequenceid=178, filesize=11.9 K 2024-11-08T00:26:21,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/665538b71af64fc9a3d9ace3896b7a3a as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/665538b71af64fc9a3d9ace3896b7a3a 2024-11-08T00:26:21,390 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/665538b71af64fc9a3d9ace3896b7a3a, entries=150, sequenceid=178, filesize=11.9 K 2024-11-08T00:26:21,390 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 22b286596054a5e4265c6f73c6acf0d1 in 919ms, sequenceid=178, compaction requested=true 2024-11-08T00:26:21,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 22b286596054a5e4265c6f73c6acf0d1: 2024-11-08T00:26:21,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22b286596054a5e4265c6f73c6acf0d1:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T00:26:21,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:26:21,390 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:26:21,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22b286596054a5e4265c6f73c6acf0d1:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T00:26:21,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:26:21,391 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:26:21,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22b286596054a5e4265c6f73c6acf0d1:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:26:21,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:26:21,392 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:26:21,392 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): 22b286596054a5e4265c6f73c6acf0d1/B is initiating minor compaction (all files) 2024-11-08T00:26:21,392 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22b286596054a5e4265c6f73c6acf0d1/B in TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:21,392 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110911 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:26:21,392 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/13b2b7cc0dc44fcaa7c435ac4237597a, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/74e17db025a74094ac3bcc0c1c0f3b1c, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/091b7dae942f4398a2be115d3796aed7] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp, totalSize=35.9 K 2024-11-08T00:26:21,392 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): 22b286596054a5e4265c6f73c6acf0d1/A is initiating minor compaction (all files) 2024-11-08T00:26:21,392 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22b286596054a5e4265c6f73c6acf0d1/A in TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:21,392 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/b7541ca9eb774e10ba90a134449f8b13, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/92704af1b09344a6bd680916e527922e, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/fa7eea9f75824ba7b5acff453eb5ab6b] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp, totalSize=108.3 K 2024-11-08T00:26:21,392 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:21,392 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. files: [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/b7541ca9eb774e10ba90a134449f8b13, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/92704af1b09344a6bd680916e527922e, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/fa7eea9f75824ba7b5acff453eb5ab6b] 2024-11-08T00:26:21,393 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 13b2b7cc0dc44fcaa7c435ac4237597a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731025579448 2024-11-08T00:26:21,393 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7541ca9eb774e10ba90a134449f8b13, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731025579448 2024-11-08T00:26:21,393 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 74e17db025a74094ac3bcc0c1c0f3b1c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731025579797 2024-11-08T00:26:21,393 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92704af1b09344a6bd680916e527922e, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731025579794 2024-11-08T00:26:21,393 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 091b7dae942f4398a2be115d3796aed7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731025580143 2024-11-08T00:26:21,394 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa7eea9f75824ba7b5acff453eb5ab6b, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731025580143 2024-11-08T00:26:21,399 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=22b286596054a5e4265c6f73c6acf0d1] 2024-11-08T00:26:21,401 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22b286596054a5e4265c6f73c6acf0d1#B#compaction#548 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:26:21,401 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/5f34c0ccbd4d4a198cdb184917d93dcc is 50, key is test_row_0/B:col10/1731025580469/Put/seqid=0 2024-11-08T00:26:21,402 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411084a02668574e145699b5ddc9fc39fe260_22b286596054a5e4265c6f73c6acf0d1 store=[table=TestAcidGuarantees family=A region=22b286596054a5e4265c6f73c6acf0d1] 2024-11-08T00:26:21,404 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411084a02668574e145699b5ddc9fc39fe260_22b286596054a5e4265c6f73c6acf0d1, store=[table=TestAcidGuarantees family=A region=22b286596054a5e4265c6f73c6acf0d1] 2024-11-08T00:26:21,404 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411084a02668574e145699b5ddc9fc39fe260_22b286596054a5e4265c6f73c6acf0d1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=22b286596054a5e4265c6f73c6acf0d1] 2024-11-08T00:26:21,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742474_1650 (size=12561) 2024-11-08T00:26:21,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742475_1651 (size=4469) 2024-11-08T00:26:21,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-08T00:26:21,577 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:26:21,578 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-08T00:26:21,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-08T00:26:21,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-08T00:26:21,580 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T00:26:21,581 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T00:26:21,581 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T00:26:21,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:21,640 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:26:21,640 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 22b286596054a5e4265c6f73c6acf0d1 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-08T00:26:21,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411080c6b074dd4b5469e939cbe22a0876c56_22b286596054a5e4265c6f73c6acf0d1 is 50, key is test_row_0/A:col10/1731025580495/Put/seqid=0 2024-11-08T00:26:21,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742476_1652 (size=14794) 2024-11-08T00:26:21,650 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:21,660 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411080c6b074dd4b5469e939cbe22a0876c56_22b286596054a5e4265c6f73c6acf0d1 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411080c6b074dd4b5469e939cbe22a0876c56_22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:21,661 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/c383773730304b03bb1ab414ad85062b, store: [table=TestAcidGuarantees family=A region=22b286596054a5e4265c6f73c6acf0d1] 2024-11-08T00:26:21,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/c383773730304b03bb1ab414ad85062b is 175, key is test_row_0/A:col10/1731025580495/Put/seqid=0 2024-11-08T00:26:21,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:21,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025641657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:21,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:21,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:21,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:21,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:21,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025641658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:21,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:21,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:21,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:21,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742477_1653 (size=39749) 2024-11-08T00:26:21,665 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=197, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/c383773730304b03bb1ab414ad85062b 2024-11-08T00:26:21,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:21,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025641661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:21,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:21,666 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:21,667 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:21,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025641662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:21,667 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:21,667 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:21,667 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:21,667 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:21,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:21,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025641663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:21,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:21,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:21,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:21,673 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/050b5d5cbf62438fbac16daebdcd1a0a is 50, key is test_row_0/B:col10/1731025580495/Put/seqid=0 2024-11-08T00:26:21,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742478_1654 (size=12151) 2024-11-08T00:26:21,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-08T00:26:21,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-08T00:26:21,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:21,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. as already flushing 2024-11-08T00:26:21,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:21,734 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:21,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:21,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:21,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:21,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025641767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:21,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:21,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:21,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:21,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:21,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025641767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:21,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:21,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:21,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:21,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:21,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025641777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:21,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:21,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025641777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:21,779 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:21,779 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:21,779 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:21,779 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:21,779 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:21,779 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:21,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:21,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025641778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:21,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:21,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:21,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:21,810 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/5f34c0ccbd4d4a198cdb184917d93dcc as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/5f34c0ccbd4d4a198cdb184917d93dcc 2024-11-08T00:26:21,811 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22b286596054a5e4265c6f73c6acf0d1#A#compaction#547 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:26:21,811 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/57a6b536c2a0456d94184a290230c2be is 175, key is test_row_0/A:col10/1731025580469/Put/seqid=0 2024-11-08T00:26:21,816 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22b286596054a5e4265c6f73c6acf0d1/B of 22b286596054a5e4265c6f73c6acf0d1 into 5f34c0ccbd4d4a198cdb184917d93dcc(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:26:21,817 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22b286596054a5e4265c6f73c6acf0d1: 2024-11-08T00:26:21,817 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., storeName=22b286596054a5e4265c6f73c6acf0d1/B, priority=13, startTime=1731025581390; duration=0sec 2024-11-08T00:26:21,817 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:26:21,817 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22b286596054a5e4265c6f73c6acf0d1:B 2024-11-08T00:26:21,817 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:26:21,818 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:26:21,818 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): 22b286596054a5e4265c6f73c6acf0d1/C is initiating minor compaction (all files) 2024-11-08T00:26:21,818 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22b286596054a5e4265c6f73c6acf0d1/C in TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:21,818 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/3c07df97f0c74d0fb599a5685a6877b9, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/8b00792d0c0d4f16bad5dd9becf4e428, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/665538b71af64fc9a3d9ace3896b7a3a] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp, totalSize=35.9 K 2024-11-08T00:26:21,818 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c07df97f0c74d0fb599a5685a6877b9, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731025579448 2024-11-08T00:26:21,819 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b00792d0c0d4f16bad5dd9becf4e428, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731025579797 2024-11-08T00:26:21,819 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 665538b71af64fc9a3d9ace3896b7a3a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731025580143 2024-11-08T00:26:21,824 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22b286596054a5e4265c6f73c6acf0d1#C#compaction#551 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:26:21,824 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/17dea555a5704a91a49a5864cc27eac1 is 50, key is test_row_0/C:col10/1731025580469/Put/seqid=0 2024-11-08T00:26:21,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742479_1655 (size=31515) 2024-11-08T00:26:21,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742480_1656 (size=12561) 2024-11-08T00:26:21,829 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/57a6b536c2a0456d94184a290230c2be as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/57a6b536c2a0456d94184a290230c2be 2024-11-08T00:26:21,833 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22b286596054a5e4265c6f73c6acf0d1/A of 22b286596054a5e4265c6f73c6acf0d1 into 57a6b536c2a0456d94184a290230c2be(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:26:21,833 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22b286596054a5e4265c6f73c6acf0d1: 2024-11-08T00:26:21,833 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., storeName=22b286596054a5e4265c6f73c6acf0d1/A, priority=13, startTime=1731025581390; duration=0sec 2024-11-08T00:26:21,833 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:26:21,833 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22b286596054a5e4265c6f73c6acf0d1:A 2024-11-08T00:26:21,886 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-08T00:26:21,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:21,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. as already flushing 2024-11-08T00:26:21,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:21,886 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:21,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:21,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:21,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-08T00:26:21,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:21,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025641977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:21,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:21,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:21,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:21,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025641978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:21,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:21,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:21,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:21,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:21,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:21,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025641987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:21,989 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:21,989 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:21,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:21,989 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:21,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025641987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:21,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:21,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025641987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:21,990 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:21,990 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:21,990 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:21,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:21,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:21,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:22,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-08T00:26:22,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:22,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. as already flushing 2024-11-08T00:26:22,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:22,039 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:22,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:22,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:22,081 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/050b5d5cbf62438fbac16daebdcd1a0a 2024-11-08T00:26:22,087 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/c32d97350ee943bb9f675314f961e94d is 50, key is test_row_0/C:col10/1731025580495/Put/seqid=0 2024-11-08T00:26:22,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742481_1657 (size=12151) 2024-11-08T00:26:22,191 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-08T00:26:22,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:22,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. as already flushing 2024-11-08T00:26:22,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:22,191 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:22,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:22,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:22,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-08T00:26:22,233 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/17dea555a5704a91a49a5864cc27eac1 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/17dea555a5704a91a49a5864cc27eac1 2024-11-08T00:26:22,239 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22b286596054a5e4265c6f73c6acf0d1/C of 22b286596054a5e4265c6f73c6acf0d1 into 17dea555a5704a91a49a5864cc27eac1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:26:22,239 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22b286596054a5e4265c6f73c6acf0d1: 2024-11-08T00:26:22,239 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., storeName=22b286596054a5e4265c6f73c6acf0d1/C, priority=13, startTime=1731025581391; duration=0sec 2024-11-08T00:26:22,239 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:26:22,239 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22b286596054a5e4265c6f73c6acf0d1:C 2024-11-08T00:26:22,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:22,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025642287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:22,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:22,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:22,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:22,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:22,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025642287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:22,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:22,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:22,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:22,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:22,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025642297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:22,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:22,300 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:22,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025642297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:22,300 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:22,300 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:22,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:22,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025642297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:22,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:22,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:22,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:22,301 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:22,301 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:22,301 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:22,344 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-08T00:26:22,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:22,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. as already flushing 2024-11-08T00:26:22,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:22,344 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:22,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:22,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:22,491 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/c32d97350ee943bb9f675314f961e94d 2024-11-08T00:26:22,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/c383773730304b03bb1ab414ad85062b as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/c383773730304b03bb1ab414ad85062b 2024-11-08T00:26:22,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-08T00:26:22,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:22,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. as already flushing 2024-11-08T00:26:22,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:22,497 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:22,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:22,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:22,500 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/c383773730304b03bb1ab414ad85062b, entries=200, sequenceid=197, filesize=38.8 K 2024-11-08T00:26:22,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/050b5d5cbf62438fbac16daebdcd1a0a as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/050b5d5cbf62438fbac16daebdcd1a0a 2024-11-08T00:26:22,503 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/050b5d5cbf62438fbac16daebdcd1a0a, entries=150, sequenceid=197, filesize=11.9 K 2024-11-08T00:26:22,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/c32d97350ee943bb9f675314f961e94d as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/c32d97350ee943bb9f675314f961e94d 2024-11-08T00:26:22,510 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/c32d97350ee943bb9f675314f961e94d, entries=150, sequenceid=197, filesize=11.9 K 2024-11-08T00:26:22,512 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 22b286596054a5e4265c6f73c6acf0d1 in 871ms, sequenceid=197, compaction requested=false 2024-11-08T00:26:22,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 22b286596054a5e4265c6f73c6acf0d1: 2024-11-08T00:26:22,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-08T00:26:22,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:22,651 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 22b286596054a5e4265c6f73c6acf0d1 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-08T00:26:22,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411087b2ec87270ff49c8ac9c558a9a69795d_22b286596054a5e4265c6f73c6acf0d1 is 50, key is test_row_0/A:col10/1731025581662/Put/seqid=0 2024-11-08T00:26:22,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742482_1658 (size=12304) 2024-11-08T00:26:22,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,664 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411087b2ec87270ff49c8ac9c558a9a69795d_22b286596054a5e4265c6f73c6acf0d1 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411087b2ec87270ff49c8ac9c558a9a69795d_22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:22,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/b87654b6d07c49a488668354d98b3ef8, store: [table=TestAcidGuarantees family=A region=22b286596054a5e4265c6f73c6acf0d1] 2024-11-08T00:26:22,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/b87654b6d07c49a488668354d98b3ef8 is 175, key is test_row_0/A:col10/1731025581662/Put/seqid=0 2024-11-08T00:26:22,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742483_1659 (size=31105) 2024-11-08T00:26:22,668 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=218, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/b87654b6d07c49a488668354d98b3ef8 2024-11-08T00:26:22,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/810f21d192084919af9625a6dedf63fc is 50, key is test_row_0/B:col10/1731025581662/Put/seqid=0 2024-11-08T00:26:22,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742484_1660 (size=12151) 2024-11-08T00:26:22,680 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/810f21d192084919af9625a6dedf63fc 2024-11-08T00:26:22,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/688a23bf04854367b7e7f0be6a1eff88 is 50, key is test_row_0/C:col10/1731025581662/Put/seqid=0 2024-11-08T00:26:22,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742485_1661 (size=12151) 2024-11-08T00:26:22,689 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/688a23bf04854367b7e7f0be6a1eff88 2024-11-08T00:26:22,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/b87654b6d07c49a488668354d98b3ef8 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/b87654b6d07c49a488668354d98b3ef8 2024-11-08T00:26:22,695 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/b87654b6d07c49a488668354d98b3ef8, entries=150, sequenceid=218, filesize=30.4 K 2024-11-08T00:26:22,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/810f21d192084919af9625a6dedf63fc as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/810f21d192084919af9625a6dedf63fc 2024-11-08T00:26:22,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,699 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/810f21d192084919af9625a6dedf63fc, entries=150, sequenceid=218, filesize=11.9 K 2024-11-08T00:26:22,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/688a23bf04854367b7e7f0be6a1eff88 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/688a23bf04854367b7e7f0be6a1eff88 2024-11-08T00:26:22,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,703 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/688a23bf04854367b7e7f0be6a1eff88, entries=150, sequenceid=218, filesize=11.9 K 2024-11-08T00:26:22,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,703 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=0 B/0 for 22b286596054a5e4265c6f73c6acf0d1 in 52ms, sequenceid=218, compaction requested=true 2024-11-08T00:26:22,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 22b286596054a5e4265c6f73c6acf0d1: 2024-11-08T00:26:22,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:22,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-08T00:26:22,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-08T00:26:22,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,706 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-08T00:26:22,706 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1240 sec 2024-11-08T00:26:22,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,717 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.1280 sec 2024-11-08T00:26:22,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-08T00:26:22,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,718 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:26:22,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,718 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-08T00:26:22,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-08T00:26:22,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-08T00:26:22,720 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T00:26:22,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,721 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T00:26:22,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,721 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T00:26:22,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-08T00:26:22,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:22,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:22,836 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:26:22,836 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 22b286596054a5e4265c6f73c6acf0d1 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T00:26:22,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411087922c1c97305451ab0a94dd239304c5d_22b286596054a5e4265c6f73c6acf0d1 is 50, key is test_row_0/A:col10/1731025582835/Put/seqid=0 2024-11-08T00:26:22,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742486_1662 (size=14794) 2024-11-08T00:26:22,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:22,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025642860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:22,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:22,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:22,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:22,872 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-08T00:26:22,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:22,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025642866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:22,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:22,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. as already flushing 2024-11-08T00:26:22,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:22,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:22,873 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:22,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:22,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:22,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025642866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:22,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:22,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:22,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:22,873 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:22,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025642867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:22,873 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:22,873 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:22,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:22,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025642867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:22,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:22,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:22,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:22,874 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:22,874 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:22,874 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:22,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:22,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:22,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025642977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:22,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:22,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025642977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:22,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:22,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:22,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:22,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:22,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025642978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:22,982 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:22,982 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:22,982 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:22,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:22,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025642978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:22,983 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:22,983 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:22,983 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:22,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:22,983 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:22,983 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:22,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025642978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:22,983 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:22,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:22,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:22,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:23,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-08T00:26:23,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:23,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. as already flushing 2024-11-08T00:26:23,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:23,026 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:23,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:23,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:23,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-08T00:26:23,180 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-08T00:26:23,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:23,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. as already flushing 2024-11-08T00:26:23,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:23,180 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:23,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:23,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:23,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:23,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025643188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:23,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:23,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:23,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:23,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:23,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025643188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:23,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:23,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:23,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:23,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:23,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025643188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:23,192 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:23,192 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:23,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:23,192 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:23,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025643188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:23,193 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:23,193 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:23,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:23,193 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:23,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025643188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:23,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:23,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:23,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:23,250 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:23,253 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411087922c1c97305451ab0a94dd239304c5d_22b286596054a5e4265c6f73c6acf0d1 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411087922c1c97305451ab0a94dd239304c5d_22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:23,253 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/6d2d6c98262348229637d037f88d700f, store: [table=TestAcidGuarantees family=A region=22b286596054a5e4265c6f73c6acf0d1] 2024-11-08T00:26:23,254 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/6d2d6c98262348229637d037f88d700f is 175, key is test_row_0/A:col10/1731025582835/Put/seqid=0 2024-11-08T00:26:23,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742487_1663 (size=39749) 2024-11-08T00:26:23,332 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-08T00:26:23,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:23,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. as already flushing 2024-11-08T00:26:23,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:23,333 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:23,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:23,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:23,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-08T00:26:23,485 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-08T00:26:23,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:23,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. as already flushing 2024-11-08T00:26:23,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:23,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:23,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:23,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:23,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:23,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025643497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:23,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:23,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:23,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:23,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:23,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025643497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:23,501 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:23,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:23,501 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:23,501 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:23,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025643498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:23,501 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:23,501 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:23,501 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:23,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:23,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025643498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:23,501 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:23,501 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:23,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:23,501 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:23,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025643498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:23,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:23,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:23,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:23,638 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-08T00:26:23,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:23,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. as already flushing 2024-11-08T00:26:23,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:23,638 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:23,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:23,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:23,658 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=229, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/6d2d6c98262348229637d037f88d700f 2024-11-08T00:26:23,663 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/7e8f0c114508427c9309f52155c7ee40 is 50, key is test_row_0/B:col10/1731025582835/Put/seqid=0 2024-11-08T00:26:23,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742488_1664 (size=12151) 2024-11-08T00:26:23,670 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/7e8f0c114508427c9309f52155c7ee40 2024-11-08T00:26:23,677 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/64be8aea7c3547b1b086e06604add343 is 50, key is test_row_0/C:col10/1731025582835/Put/seqid=0 2024-11-08T00:26:23,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742489_1665 (size=12151) 2024-11-08T00:26:23,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-08T00:26:23,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:23,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. as already flushing 2024-11-08T00:26:23,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:23,791 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:23,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:23,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:23,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-08T00:26:23,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-08T00:26:23,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:23,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. as already flushing 2024-11-08T00:26:23,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:23,944 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:23,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:23,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:24,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:24,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025644007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:24,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:24,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:24,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:24,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:24,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025644007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:24,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:24,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:24,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:24,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:24,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025644007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:24,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:24,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025644008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:24,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:24,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:24,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:24,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:24,011 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:24,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025644008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:24,011 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:24,011 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:24,012 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:24,012 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:24,012 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:24,085 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/64be8aea7c3547b1b086e06604add343 2024-11-08T00:26:24,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/6d2d6c98262348229637d037f88d700f as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/6d2d6c98262348229637d037f88d700f 2024-11-08T00:26:24,092 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/6d2d6c98262348229637d037f88d700f, entries=200, sequenceid=229, filesize=38.8 K 2024-11-08T00:26:24,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/7e8f0c114508427c9309f52155c7ee40 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/7e8f0c114508427c9309f52155c7ee40 2024-11-08T00:26:24,095 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/7e8f0c114508427c9309f52155c7ee40, entries=150, sequenceid=229, filesize=11.9 K 2024-11-08T00:26:24,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-08T00:26:24,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:24,096 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/64be8aea7c3547b1b086e06604add343 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/64be8aea7c3547b1b086e06604add343 2024-11-08T00:26:24,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. as already flushing 2024-11-08T00:26:24,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:24,096 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:24,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:24,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:24,100 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/64be8aea7c3547b1b086e06604add343, entries=150, sequenceid=229, filesize=11.9 K 2024-11-08T00:26:24,100 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 22b286596054a5e4265c6f73c6acf0d1 in 1264ms, sequenceid=229, compaction requested=true 2024-11-08T00:26:24,100 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 22b286596054a5e4265c6f73c6acf0d1: 2024-11-08T00:26:24,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22b286596054a5e4265c6f73c6acf0d1:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T00:26:24,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:26:24,100 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T00:26:24,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22b286596054a5e4265c6f73c6acf0d1:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T00:26:24,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:26:24,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22b286596054a5e4265c6f73c6acf0d1:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:26:24,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-08T00:26:24,100 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T00:26:24,101 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142118 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T00:26:24,101 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): 22b286596054a5e4265c6f73c6acf0d1/A is initiating minor compaction (all files) 2024-11-08T00:26:24,101 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22b286596054a5e4265c6f73c6acf0d1/A in TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:24,101 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/57a6b536c2a0456d94184a290230c2be, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/c383773730304b03bb1ab414ad85062b, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/b87654b6d07c49a488668354d98b3ef8, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/6d2d6c98262348229637d037f88d700f] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp, totalSize=138.8 K 2024-11-08T00:26:24,101 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:24,101 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. files: [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/57a6b536c2a0456d94184a290230c2be, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/c383773730304b03bb1ab414ad85062b, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/b87654b6d07c49a488668354d98b3ef8, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/6d2d6c98262348229637d037f88d700f] 2024-11-08T00:26:24,102 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T00:26:24,102 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): 22b286596054a5e4265c6f73c6acf0d1/B is initiating minor compaction (all files) 2024-11-08T00:26:24,102 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22b286596054a5e4265c6f73c6acf0d1/B in TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:24,102 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/5f34c0ccbd4d4a198cdb184917d93dcc, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/050b5d5cbf62438fbac16daebdcd1a0a, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/810f21d192084919af9625a6dedf63fc, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/7e8f0c114508427c9309f52155c7ee40] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp, totalSize=47.9 K 2024-11-08T00:26:24,102 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f34c0ccbd4d4a198cdb184917d93dcc, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731025580143 2024-11-08T00:26:24,102 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57a6b536c2a0456d94184a290230c2be, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731025580143 2024-11-08T00:26:24,102 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 050b5d5cbf62438fbac16daebdcd1a0a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731025580493 2024-11-08T00:26:24,103 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting c383773730304b03bb1ab414ad85062b, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731025580493 2024-11-08T00:26:24,103 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 810f21d192084919af9625a6dedf63fc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731025581657 2024-11-08T00:26:24,103 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting b87654b6d07c49a488668354d98b3ef8, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731025581657 2024-11-08T00:26:24,103 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e8f0c114508427c9309f52155c7ee40, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731025582833 2024-11-08T00:26:24,103 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d2d6c98262348229637d037f88d700f, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731025582810 2024-11-08T00:26:24,108 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=22b286596054a5e4265c6f73c6acf0d1] 2024-11-08T00:26:24,109 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22b286596054a5e4265c6f73c6acf0d1#B#compaction#559 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:26:24,110 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/73b5f4b3ad87428a906d745e8e4edeae is 50, key is test_row_0/B:col10/1731025582835/Put/seqid=0 2024-11-08T00:26:24,110 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241108963a0eb4812f49879c701ac68ad098e3_22b286596054a5e4265c6f73c6acf0d1 store=[table=TestAcidGuarantees family=A region=22b286596054a5e4265c6f73c6acf0d1] 2024-11-08T00:26:24,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742490_1666 (size=12697) 2024-11-08T00:26:24,121 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241108963a0eb4812f49879c701ac68ad098e3_22b286596054a5e4265c6f73c6acf0d1, store=[table=TestAcidGuarantees family=A region=22b286596054a5e4265c6f73c6acf0d1] 2024-11-08T00:26:24,121 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108963a0eb4812f49879c701ac68ad098e3_22b286596054a5e4265c6f73c6acf0d1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=22b286596054a5e4265c6f73c6acf0d1] 2024-11-08T00:26:24,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742491_1667 (size=4469) 2024-11-08T00:26:24,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-08T00:26:24,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:24,250 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 22b286596054a5e4265c6f73c6acf0d1 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-08T00:26:24,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411082765f563d7174fcfac56d4843f1b5716_22b286596054a5e4265c6f73c6acf0d1 is 50, key is test_row_0/A:col10/1731025582860/Put/seqid=0 2024-11-08T00:26:24,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742492_1668 (size=12304) 2024-11-08T00:26:24,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:24,274 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411082765f563d7174fcfac56d4843f1b5716_22b286596054a5e4265c6f73c6acf0d1 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411082765f563d7174fcfac56d4843f1b5716_22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:24,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/54f2c2b88c9949a8924a8adc7f85ff29, store: [table=TestAcidGuarantees family=A region=22b286596054a5e4265c6f73c6acf0d1] 2024-11-08T00:26:24,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/54f2c2b88c9949a8924a8adc7f85ff29 is 175, key is test_row_0/A:col10/1731025582860/Put/seqid=0 2024-11-08T00:26:24,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742493_1669 (size=31105) 2024-11-08T00:26:24,525 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/73b5f4b3ad87428a906d745e8e4edeae as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/73b5f4b3ad87428a906d745e8e4edeae 2024-11-08T00:26:24,526 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22b286596054a5e4265c6f73c6acf0d1#A#compaction#560 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:26:24,527 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/b40f154deef04282a438b416dd15bbbc is 175, key is test_row_0/A:col10/1731025582835/Put/seqid=0 2024-11-08T00:26:24,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742494_1670 (size=31651) 2024-11-08T00:26:24,534 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 22b286596054a5e4265c6f73c6acf0d1/B of 22b286596054a5e4265c6f73c6acf0d1 into 73b5f4b3ad87428a906d745e8e4edeae(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:26:24,534 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22b286596054a5e4265c6f73c6acf0d1: 2024-11-08T00:26:24,534 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., storeName=22b286596054a5e4265c6f73c6acf0d1/B, priority=12, startTime=1731025584100; duration=0sec 2024-11-08T00:26:24,534 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:26:24,534 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22b286596054a5e4265c6f73c6acf0d1:B 2024-11-08T00:26:24,534 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T00:26:24,537 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T00:26:24,537 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): 22b286596054a5e4265c6f73c6acf0d1/C is initiating minor compaction (all files) 2024-11-08T00:26:24,537 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22b286596054a5e4265c6f73c6acf0d1/C in TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:24,537 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/17dea555a5704a91a49a5864cc27eac1, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/c32d97350ee943bb9f675314f961e94d, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/688a23bf04854367b7e7f0be6a1eff88, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/64be8aea7c3547b1b086e06604add343] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp, totalSize=47.9 K 2024-11-08T00:26:24,537 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 17dea555a5704a91a49a5864cc27eac1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731025580143 2024-11-08T00:26:24,538 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting c32d97350ee943bb9f675314f961e94d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731025580493 2024-11-08T00:26:24,538 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 688a23bf04854367b7e7f0be6a1eff88, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731025581657 2024-11-08T00:26:24,538 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 64be8aea7c3547b1b086e06604add343, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731025582833 2024-11-08T00:26:24,540 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/b40f154deef04282a438b416dd15bbbc as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/b40f154deef04282a438b416dd15bbbc 2024-11-08T00:26:24,545 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 22b286596054a5e4265c6f73c6acf0d1/A of 22b286596054a5e4265c6f73c6acf0d1 into b40f154deef04282a438b416dd15bbbc(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:26:24,545 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22b286596054a5e4265c6f73c6acf0d1: 2024-11-08T00:26:24,545 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., storeName=22b286596054a5e4265c6f73c6acf0d1/A, priority=12, startTime=1731025584100; duration=0sec 2024-11-08T00:26:24,545 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:26:24,545 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22b286596054a5e4265c6f73c6acf0d1:A 2024-11-08T00:26:24,545 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22b286596054a5e4265c6f73c6acf0d1#C#compaction#562 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:26:24,546 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/e76c092b0bd94d748a244cb6597d5fda is 50, key is test_row_0/C:col10/1731025582835/Put/seqid=0 2024-11-08T00:26:24,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742495_1671 (size=12697) 2024-11-08T00:26:24,684 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=255, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/54f2c2b88c9949a8924a8adc7f85ff29 2024-11-08T00:26:24,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/e02d9475bbfd4d46a54efebf474e7b24 is 50, key is test_row_0/B:col10/1731025582860/Put/seqid=0 2024-11-08T00:26:24,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742496_1672 (size=12151) 2024-11-08T00:26:24,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-08T00:26:24,953 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/e76c092b0bd94d748a244cb6597d5fda as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/e76c092b0bd94d748a244cb6597d5fda 2024-11-08T00:26:24,957 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 22b286596054a5e4265c6f73c6acf0d1/C of 22b286596054a5e4265c6f73c6acf0d1 into e76c092b0bd94d748a244cb6597d5fda(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:26:24,957 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22b286596054a5e4265c6f73c6acf0d1: 2024-11-08T00:26:24,957 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., storeName=22b286596054a5e4265c6f73c6acf0d1/C, priority=12, startTime=1731025584100; duration=0sec 2024-11-08T00:26:24,957 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:26:24,957 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22b286596054a5e4265c6f73c6acf0d1:C 2024-11-08T00:26:25,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:25,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. as already flushing 2024-11-08T00:26:25,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:25,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025645029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:25,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:25,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:25,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:25,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:25,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025645029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:25,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:25,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:25,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:25,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:25,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025645030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:25,034 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:25,034 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:25,034 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:25,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:25,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:25,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025645034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:25,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025645034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:25,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:25,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:25,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:25,038 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:25,038 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:25,038 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:25,098 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/e02d9475bbfd4d46a54efebf474e7b24 2024-11-08T00:26:25,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/9c6c926bd4f6441f868048e6a492648e is 50, key is test_row_0/C:col10/1731025582860/Put/seqid=0 2024-11-08T00:26:25,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742497_1673 (size=12151) 2024-11-08T00:26:25,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:25,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025645138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:25,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:25,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025645138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:25,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:25,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:25,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:25,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:25,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:25,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:25,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:25,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025645138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:25,141 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:25,141 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:25,141 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:25,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:25,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025645148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:25,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:25,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:25,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:25,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:25,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025645148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:25,153 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:25,153 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:25,153 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:25,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:25,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025645347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:25,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:25,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:25,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:25,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:25,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025645348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:25,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:25,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025645348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:25,349 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:25,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:25,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:25,349 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:25,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:25,349 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:25,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:25,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025645357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:25,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:25,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:25,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:25,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025645357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:25,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:25,360 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:25,360 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:25,360 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:25,539 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/9c6c926bd4f6441f868048e6a492648e 2024-11-08T00:26:25,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/54f2c2b88c9949a8924a8adc7f85ff29 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/54f2c2b88c9949a8924a8adc7f85ff29 2024-11-08T00:26:25,557 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/54f2c2b88c9949a8924a8adc7f85ff29, entries=150, sequenceid=255, filesize=30.4 K 2024-11-08T00:26:25,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/e02d9475bbfd4d46a54efebf474e7b24 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/e02d9475bbfd4d46a54efebf474e7b24 2024-11-08T00:26:25,568 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/e02d9475bbfd4d46a54efebf474e7b24, entries=150, sequenceid=255, filesize=11.9 K 2024-11-08T00:26:25,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/9c6c926bd4f6441f868048e6a492648e as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/9c6c926bd4f6441f868048e6a492648e 2024-11-08T00:26:25,573 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/9c6c926bd4f6441f868048e6a492648e, entries=150, sequenceid=255, filesize=11.9 K 2024-11-08T00:26:25,574 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 22b286596054a5e4265c6f73c6acf0d1 in 1325ms, sequenceid=255, compaction requested=false 2024-11-08T00:26:25,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 22b286596054a5e4265c6f73c6acf0d1: 2024-11-08T00:26:25,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:25,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-08T00:26:25,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-08T00:26:25,578 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-08T00:26:25,579 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8560 sec 2024-11-08T00:26:25,588 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 2.8650 sec 2024-11-08T00:26:25,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:25,665 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:26:25,665 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 22b286596054a5e4265c6f73c6acf0d1 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-08T00:26:25,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110881ad7bb350a94dafa259339fa3c89829_22b286596054a5e4265c6f73c6acf0d1 is 50, key is test_row_0/A:col10/1731025585659/Put/seqid=0 2024-11-08T00:26:25,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742498_1674 (size=17534) 2024-11-08T00:26:25,721 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:25,724 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110881ad7bb350a94dafa259339fa3c89829_22b286596054a5e4265c6f73c6acf0d1 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110881ad7bb350a94dafa259339fa3c89829_22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:25,725 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/000864556f4748668d3deb8a9ad42d96, store: [table=TestAcidGuarantees family=A region=22b286596054a5e4265c6f73c6acf0d1] 2024-11-08T00:26:25,725 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/000864556f4748668d3deb8a9ad42d96 is 175, key is test_row_0/A:col10/1731025585659/Put/seqid=0 2024-11-08T00:26:25,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742499_1675 (size=48639) 2024-11-08T00:26:25,730 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=272, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/000864556f4748668d3deb8a9ad42d96 2024-11-08T00:26:25,736 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/829365a4219a4b47977b2e8d6c50c014 is 50, key is test_row_0/B:col10/1731025585659/Put/seqid=0 2024-11-08T00:26:25,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742500_1676 (size=12301) 2024-11-08T00:26:25,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:25,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:25,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025645764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:25,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025645765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:25,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:25,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025645772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:25,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:25,777 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:25,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025645772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:25,777 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:25,777 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:25,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:25,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:25,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:25,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:25,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:25,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:25,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025645772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:25,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:25,777 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:25,777 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:25,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:25,777 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:25,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:25,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:25,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:25,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025645887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:25,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:25,889 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:25,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025645887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:25,889 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:25,889 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:25,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:25,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:25,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025645887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:25,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:25,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:25,889 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:25,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:25,889 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:25,889 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:25,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025645887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:25,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:25,890 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:25,890 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:25,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025645887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:25,890 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:25,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:25,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:25,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:26,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:26,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025646097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:26,101 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:26,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:26,101 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:26,101 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:26,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025646097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:26,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:26,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:26,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:26,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:26,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025646097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:26,101 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:26,101 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:26,101 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:26,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:26,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025646097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:26,101 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:26,101 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:26,101 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:26,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:26,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025646097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:26,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:26,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:26,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:26,161 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/829365a4219a4b47977b2e8d6c50c014 2024-11-08T00:26:26,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/ddd9a4ef90644ec4aab4e8e449c93cb3 is 50, key is test_row_0/C:col10/1731025585659/Put/seqid=0 2024-11-08T00:26:26,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742501_1677 (size=12301) 2024-11-08T00:26:26,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:26,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025646407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:26,411 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:26,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:26,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:26,411 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:26,411 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:26,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025646407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:26,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025646407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:26,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:26,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:26,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:26,411 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:26,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:26,411 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:26,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:26,411 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:26,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025646407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:26,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025646407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:26,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:26,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:26,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:26,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:26,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:26,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:26,575 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/ddd9a4ef90644ec4aab4e8e449c93cb3 2024-11-08T00:26:26,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/000864556f4748668d3deb8a9ad42d96 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/000864556f4748668d3deb8a9ad42d96 2024-11-08T00:26:26,581 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/000864556f4748668d3deb8a9ad42d96, entries=250, sequenceid=272, filesize=47.5 K 2024-11-08T00:26:26,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/829365a4219a4b47977b2e8d6c50c014 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/829365a4219a4b47977b2e8d6c50c014 2024-11-08T00:26:26,584 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/829365a4219a4b47977b2e8d6c50c014, entries=150, sequenceid=272, filesize=12.0 K 2024-11-08T00:26:26,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/ddd9a4ef90644ec4aab4e8e449c93cb3 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/ddd9a4ef90644ec4aab4e8e449c93cb3 2024-11-08T00:26:26,587 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/ddd9a4ef90644ec4aab4e8e449c93cb3, entries=150, sequenceid=272, filesize=12.0 K 2024-11-08T00:26:26,588 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 22b286596054a5e4265c6f73c6acf0d1 in 923ms, sequenceid=272, compaction requested=true 2024-11-08T00:26:26,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 22b286596054a5e4265c6f73c6acf0d1: 2024-11-08T00:26:26,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22b286596054a5e4265c6f73c6acf0d1:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T00:26:26,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:26:26,588 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:26:26,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22b286596054a5e4265c6f73c6acf0d1:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T00:26:26,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:26:26,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22b286596054a5e4265c6f73c6acf0d1:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:26:26,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:26:26,588 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:26:26,589 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111395 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:26:26,589 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:26:26,589 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): 22b286596054a5e4265c6f73c6acf0d1/B is initiating minor compaction (all files) 2024-11-08T00:26:26,589 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): 22b286596054a5e4265c6f73c6acf0d1/A is initiating minor compaction (all files) 2024-11-08T00:26:26,589 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22b286596054a5e4265c6f73c6acf0d1/A in TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:26,589 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22b286596054a5e4265c6f73c6acf0d1/B in TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:26,589 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/73b5f4b3ad87428a906d745e8e4edeae, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/e02d9475bbfd4d46a54efebf474e7b24, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/829365a4219a4b47977b2e8d6c50c014] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp, totalSize=36.3 K 2024-11-08T00:26:26,589 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/b40f154deef04282a438b416dd15bbbc, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/54f2c2b88c9949a8924a8adc7f85ff29, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/000864556f4748668d3deb8a9ad42d96] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp, totalSize=108.8 K 2024-11-08T00:26:26,589 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:26,589 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. files: [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/b40f154deef04282a438b416dd15bbbc, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/54f2c2b88c9949a8924a8adc7f85ff29, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/000864556f4748668d3deb8a9ad42d96] 2024-11-08T00:26:26,589 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 73b5f4b3ad87428a906d745e8e4edeae, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731025582833 2024-11-08T00:26:26,589 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting b40f154deef04282a438b416dd15bbbc, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731025582833 2024-11-08T00:26:26,589 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting e02d9475bbfd4d46a54efebf474e7b24, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731025582860 2024-11-08T00:26:26,590 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54f2c2b88c9949a8924a8adc7f85ff29, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731025582860 2024-11-08T00:26:26,590 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 829365a4219a4b47977b2e8d6c50c014, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731025585032 2024-11-08T00:26:26,596 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 000864556f4748668d3deb8a9ad42d96, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731025585032 2024-11-08T00:26:26,601 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=22b286596054a5e4265c6f73c6acf0d1] 2024-11-08T00:26:26,602 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22b286596054a5e4265c6f73c6acf0d1#B#compaction#568 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:26:26,602 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/1c8ff002dc2f4d6ea0605bc759da8f27 is 50, key is test_row_0/B:col10/1731025585659/Put/seqid=0 2024-11-08T00:26:26,604 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110892165e0b3f0e48fb82b8a71b5c6f9c12_22b286596054a5e4265c6f73c6acf0d1 store=[table=TestAcidGuarantees family=A region=22b286596054a5e4265c6f73c6acf0d1] 2024-11-08T00:26:26,606 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110892165e0b3f0e48fb82b8a71b5c6f9c12_22b286596054a5e4265c6f73c6acf0d1, store=[table=TestAcidGuarantees family=A region=22b286596054a5e4265c6f73c6acf0d1] 2024-11-08T00:26:26,606 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110892165e0b3f0e48fb82b8a71b5c6f9c12_22b286596054a5e4265c6f73c6acf0d1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=22b286596054a5e4265c6f73c6acf0d1] 2024-11-08T00:26:26,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742502_1678 (size=12949) 2024-11-08T00:26:26,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742503_1679 (size=4469) 2024-11-08T00:26:26,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-08T00:26:26,888 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:26:26,888 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-08T00:26:26,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-08T00:26:26,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-08T00:26:26,889 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T00:26:26,890 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T00:26:26,890 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T00:26:26,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:26,920 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:26:26,920 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 22b286596054a5e4265c6f73c6acf0d1 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-08T00:26:26,928 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411086561f7f4c823487ea41595bf134e3246_22b286596054a5e4265c6f73c6acf0d1 is 50, key is test_row_0/A:col10/1731025586919/Put/seqid=0 2024-11-08T00:26:26,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742504_1680 (size=12454) 2024-11-08T00:26:26,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:26,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025646929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:26,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:26,937 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:26,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:26,937 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:26,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025646929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:26,937 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:26,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025646930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:26,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:26,937 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:26,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:26,937 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:26,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025646930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:26,937 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:26,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025646931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:26,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:26,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:26,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:26,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:26,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:26,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:26,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:26,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:26,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:26,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-08T00:26:27,022 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22b286596054a5e4265c6f73c6acf0d1#A#compaction#569 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:26:27,023 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/6a227cc8a85b406ea3667e692ab65bf7 is 175, key is test_row_0/A:col10/1731025585659/Put/seqid=0 2024-11-08T00:26:27,024 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/1c8ff002dc2f4d6ea0605bc759da8f27 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/1c8ff002dc2f4d6ea0605bc759da8f27 2024-11-08T00:26:27,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742505_1681 (size=31903) 2024-11-08T00:26:27,032 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22b286596054a5e4265c6f73c6acf0d1/B of 22b286596054a5e4265c6f73c6acf0d1 into 1c8ff002dc2f4d6ea0605bc759da8f27(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:26:27,032 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22b286596054a5e4265c6f73c6acf0d1: 2024-11-08T00:26:27,032 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., storeName=22b286596054a5e4265c6f73c6acf0d1/B, priority=13, startTime=1731025586588; duration=0sec 2024-11-08T00:26:27,032 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:26:27,032 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22b286596054a5e4265c6f73c6acf0d1:B 2024-11-08T00:26:27,032 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:26:27,033 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/6a227cc8a85b406ea3667e692ab65bf7 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/6a227cc8a85b406ea3667e692ab65bf7 2024-11-08T00:26:27,034 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:26:27,034 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): 22b286596054a5e4265c6f73c6acf0d1/C is initiating minor compaction (all files) 2024-11-08T00:26:27,034 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22b286596054a5e4265c6f73c6acf0d1/C in TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:27,034 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/e76c092b0bd94d748a244cb6597d5fda, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/9c6c926bd4f6441f868048e6a492648e, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/ddd9a4ef90644ec4aab4e8e449c93cb3] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp, totalSize=36.3 K 2024-11-08T00:26:27,034 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting e76c092b0bd94d748a244cb6597d5fda, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731025582833 2024-11-08T00:26:27,034 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c6c926bd4f6441f868048e6a492648e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731025582860 2024-11-08T00:26:27,035 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting ddd9a4ef90644ec4aab4e8e449c93cb3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731025585032 2024-11-08T00:26:27,038 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22b286596054a5e4265c6f73c6acf0d1/A of 22b286596054a5e4265c6f73c6acf0d1 into 6a227cc8a85b406ea3667e692ab65bf7(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:26:27,038 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22b286596054a5e4265c6f73c6acf0d1: 2024-11-08T00:26:27,038 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., storeName=22b286596054a5e4265c6f73c6acf0d1/A, priority=13, startTime=1731025586588; duration=0sec 2024-11-08T00:26:27,038 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:26:27,038 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22b286596054a5e4265c6f73c6acf0d1:A 2024-11-08T00:26:27,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-08T00:26:27,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:27,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. as already flushing 2024-11-08T00:26:27,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:27,042 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:27,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:27,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:27,043 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22b286596054a5e4265c6f73c6acf0d1#C#compaction#571 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:26:27,043 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/52b8c034c60c4de783fca133084ce0bd is 50, key is test_row_0/C:col10/1731025585659/Put/seqid=0 2024-11-08T00:26:27,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742506_1682 (size=12949) 2024-11-08T00:26:27,050 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/52b8c034c60c4de783fca133084ce0bd as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/52b8c034c60c4de783fca133084ce0bd 2024-11-08T00:26:27,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:27,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025647047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:27,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:27,051 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:27,051 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:27,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025647047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:27,051 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:27,051 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:27,051 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:27,051 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:27,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:27,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025647047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:27,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:27,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:27,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:27,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:27,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025647048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:27,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:27,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:27,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:27,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025647048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:27,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:27,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:27,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:27,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:27,055 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22b286596054a5e4265c6f73c6acf0d1/C of 22b286596054a5e4265c6f73c6acf0d1 into 52b8c034c60c4de783fca133084ce0bd(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:26:27,055 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22b286596054a5e4265c6f73c6acf0d1: 2024-11-08T00:26:27,055 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., storeName=22b286596054a5e4265c6f73c6acf0d1/C, priority=13, startTime=1731025586588; duration=0sec 2024-11-08T00:26:27,055 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:26:27,055 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22b286596054a5e4265c6f73c6acf0d1:C 2024-11-08T00:26:27,194 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-08T00:26:27,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:27,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. as already flushing 2024-11-08T00:26:27,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:27,194 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:27,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:27,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:27,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-08T00:26:27,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:27,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:27,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025647257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:27,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025647258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:27,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:27,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:27,260 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:27,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025647258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:27,260 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:27,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025647258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:27,260 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:27,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:27,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:27,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025647258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:27,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:27,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:27,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:27,261 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:27,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:27,261 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:27,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:27,261 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:27,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:27,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:27,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:27,331 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:27,334 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411086561f7f4c823487ea41595bf134e3246_22b286596054a5e4265c6f73c6acf0d1 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411086561f7f4c823487ea41595bf134e3246_22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:27,334 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/fc671ddb22a145a6ba5929fc0eedc7a8, store: [table=TestAcidGuarantees family=A region=22b286596054a5e4265c6f73c6acf0d1] 2024-11-08T00:26:27,335 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/fc671ddb22a145a6ba5929fc0eedc7a8 is 175, key is test_row_0/A:col10/1731025586919/Put/seqid=0 2024-11-08T00:26:27,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742507_1683 (size=31255) 2024-11-08T00:26:27,350 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-08T00:26:27,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:27,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. as already flushing 2024-11-08T00:26:27,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:27,350 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:27,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:27,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:27,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-08T00:26:27,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:27,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. as already flushing 2024-11-08T00:26:27,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:27,503 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:27,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:27,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:27,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-08T00:26:27,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:27,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025647568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:27,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:27,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025647568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:27,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:27,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:27,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025647568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:27,571 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:27,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:27,571 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:27,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:27,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:27,571 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:27,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025647568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:27,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:27,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025647568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:27,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:27,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:27,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:27,571 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:27,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:27,571 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:27,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:27,571 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:27,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:27,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-08T00:26:27,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:27,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. as already flushing 2024-11-08T00:26:27,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:27,656 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:27,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:27,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:27,738 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=298, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/fc671ddb22a145a6ba5929fc0eedc7a8 2024-11-08T00:26:27,744 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/824b72f7100645299f15f31b8e6f1450 is 50, key is test_row_0/B:col10/1731025586919/Put/seqid=0 2024-11-08T00:26:27,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742508_1684 (size=12301) 2024-11-08T00:26:27,759 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/824b72f7100645299f15f31b8e6f1450 2024-11-08T00:26:27,766 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/03a46c21944a4b56af542ea48a43ced3 is 50, key is test_row_0/C:col10/1731025586919/Put/seqid=0 2024-11-08T00:26:27,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742509_1685 (size=12301) 2024-11-08T00:26:27,772 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/03a46c21944a4b56af542ea48a43ced3 2024-11-08T00:26:27,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/fc671ddb22a145a6ba5929fc0eedc7a8 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/fc671ddb22a145a6ba5929fc0eedc7a8 2024-11-08T00:26:27,780 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/fc671ddb22a145a6ba5929fc0eedc7a8, entries=150, sequenceid=298, filesize=30.5 K 2024-11-08T00:26:27,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/824b72f7100645299f15f31b8e6f1450 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/824b72f7100645299f15f31b8e6f1450 2024-11-08T00:26:27,784 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/824b72f7100645299f15f31b8e6f1450, entries=150, sequenceid=298, filesize=12.0 K 2024-11-08T00:26:27,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/03a46c21944a4b56af542ea48a43ced3 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/03a46c21944a4b56af542ea48a43ced3 2024-11-08T00:26:27,787 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/03a46c21944a4b56af542ea48a43ced3, entries=150, sequenceid=298, filesize=12.0 K 2024-11-08T00:26:27,788 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 22b286596054a5e4265c6f73c6acf0d1 in 868ms, sequenceid=298, compaction requested=false 2024-11-08T00:26:27,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 22b286596054a5e4265c6f73c6acf0d1: 2024-11-08T00:26:27,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-08T00:26:27,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:27,809 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 22b286596054a5e4265c6f73c6acf0d1 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-08T00:26:27,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108adae130fabea4c168457d2b769ce6025_22b286596054a5e4265c6f73c6acf0d1 is 50, key is test_row_0/A:col10/1731025586926/Put/seqid=0 2024-11-08T00:26:27,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742510_1686 (size=12454) 2024-11-08T00:26:28,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-08T00:26:28,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:28,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. as already flushing 2024-11-08T00:26:28,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:28,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025648100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:28,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:28,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:28,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025648100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:28,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:28,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:28,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:28,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:28,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:28,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:28,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025648104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:28,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:28,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:28,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:28,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:28,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025648105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:28,108 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:28,108 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:28,108 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:28,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:28,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025648106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:28,111 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:28,111 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:28,111 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:28,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:28,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:28,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025648217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:28,221 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:28,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:28,221 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:28,221 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:28,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025648217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:28,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:28,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025648217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:28,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:28,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:28,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:28,221 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108adae130fabea4c168457d2b769ce6025_22b286596054a5e4265c6f73c6acf0d1 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108adae130fabea4c168457d2b769ce6025_22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:28,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:28,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:28,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:28,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:28,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025648217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:28,222 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:28,222 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:28,222 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:28,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:28,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025648217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:28,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/0edbab4084c9499598678c26cb72e2c0, store: [table=TestAcidGuarantees family=A region=22b286596054a5e4265c6f73c6acf0d1] 2024-11-08T00:26:28,222 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:28,222 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:28,222 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:28,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/0edbab4084c9499598678c26cb72e2c0 is 175, key is test_row_0/A:col10/1731025586926/Put/seqid=0 2024-11-08T00:26:28,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742511_1687 (size=31255) 2024-11-08T00:26:28,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:28,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025648427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:28,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:28,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:28,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025648427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:28,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:28,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:28,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:28,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:28,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:28,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:28,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025648427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:28,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:28,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:28,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:28,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:28,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025648427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:28,430 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:28,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:28,430 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:28,430 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:28,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025648427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:28,430 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:28,431 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:28,431 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:28,625 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=311, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/0edbab4084c9499598678c26cb72e2c0 2024-11-08T00:26:28,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/23a604109da64446ae3066710e77cb8c is 50, key is test_row_0/B:col10/1731025586926/Put/seqid=0 2024-11-08T00:26:28,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742512_1688 (size=12301) 2024-11-08T00:26:28,634 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/23a604109da64446ae3066710e77cb8c 2024-11-08T00:26:28,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/368e2e2b21154bc491d609bd195f55d7 is 50, key is test_row_0/C:col10/1731025586926/Put/seqid=0 2024-11-08T00:26:28,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742513_1689 (size=12301) 2024-11-08T00:26:28,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:28,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025648737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:28,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:28,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:28,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:28,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025648737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:28,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:28,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:28,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:28,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:28,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:28,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025648738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:28,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:28,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:28,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:28,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:28,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025648738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:28,740 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:28,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:28,740 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:28,740 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:28,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025648738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:28,741 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:28,741 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:28,741 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:29,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-08T00:26:29,043 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/368e2e2b21154bc491d609bd195f55d7 2024-11-08T00:26:29,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/0edbab4084c9499598678c26cb72e2c0 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/0edbab4084c9499598678c26cb72e2c0 2024-11-08T00:26:29,049 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/0edbab4084c9499598678c26cb72e2c0, entries=150, sequenceid=311, filesize=30.5 K 2024-11-08T00:26:29,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/23a604109da64446ae3066710e77cb8c as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/23a604109da64446ae3066710e77cb8c 2024-11-08T00:26:29,052 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/23a604109da64446ae3066710e77cb8c, entries=150, sequenceid=311, filesize=12.0 K 2024-11-08T00:26:29,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/368e2e2b21154bc491d609bd195f55d7 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/368e2e2b21154bc491d609bd195f55d7 2024-11-08T00:26:29,055 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/368e2e2b21154bc491d609bd195f55d7, entries=150, sequenceid=311, filesize=12.0 K 2024-11-08T00:26:29,056 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 22b286596054a5e4265c6f73c6acf0d1 in 1247ms, sequenceid=311, compaction requested=true 2024-11-08T00:26:29,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 22b286596054a5e4265c6f73c6acf0d1: 2024-11-08T00:26:29,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:29,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-08T00:26:29,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-08T00:26:29,059 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-08T00:26:29,059 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1670 sec 2024-11-08T00:26:29,060 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 2.1710 sec 2024-11-08T00:26:29,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:29,250 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:26:29,250 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 22b286596054a5e4265c6f73c6acf0d1 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-08T00:26:29,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:29,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025649253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:29,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:29,257 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:29,257 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:29,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025649253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:29,257 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:29,257 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:29,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:29,257 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:29,257 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:29,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025649253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:29,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:29,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:29,257 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110876930a9370eb4074a25753803aaed8a5_22b286596054a5e4265c6f73c6acf0d1 is 50, key is test_row_0/A:col10/1731025588105/Put/seqid=0 2024-11-08T00:26:29,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:29,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742514_1690 (size=12454) 2024-11-08T00:26:29,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:29,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025649257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:29,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:29,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025649257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:29,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:29,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:29,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:29,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:29,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:29,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:29,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:29,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025649367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:29,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:29,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:29,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:29,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025649367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:29,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:29,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:29,370 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:29,370 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:29,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025649367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:29,370 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:29,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:29,370 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:29,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025649367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:29,370 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:29,370 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:29,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:29,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:29,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:29,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:29,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025649367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:29,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:29,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:29,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:29,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:29,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025649577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:29,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:29,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:29,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:29,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025649577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:29,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:29,579 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:29,579 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:29,579 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:29,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:29,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025649577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:29,579 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:29,579 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:29,579 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:29,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:29,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025649577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:29,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:29,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:29,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:29,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:29,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025649577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:29,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:29,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:29,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:29,661 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,664 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110876930a9370eb4074a25753803aaed8a5_22b286596054a5e4265c6f73c6acf0d1 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110876930a9370eb4074a25753803aaed8a5_22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:29,665 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/b79df6fe14f74e7d95471dc2bbdad9a6, store: [table=TestAcidGuarantees family=A region=22b286596054a5e4265c6f73c6acf0d1] 2024-11-08T00:26:29,665 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/b79df6fe14f74e7d95471dc2bbdad9a6 is 175, key is test_row_0/A:col10/1731025588105/Put/seqid=0 2024-11-08T00:26:29,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742515_1691 (size=31255) 2024-11-08T00:26:29,669 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=339, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/b79df6fe14f74e7d95471dc2bbdad9a6 2024-11-08T00:26:29,676 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/ff1156af08a5411b890669705e4326d5 is 50, key is test_row_0/B:col10/1731025588105/Put/seqid=0 2024-11-08T00:26:29,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742516_1692 (size=12301) 2024-11-08T00:26:29,679 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/ff1156af08a5411b890669705e4326d5 2024-11-08T00:26:29,685 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/db2b44e044af4e31a347b7cc89501de6 is 50, key is test_row_0/C:col10/1731025588105/Put/seqid=0 2024-11-08T00:26:29,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742517_1693 (size=12301) 2024-11-08T00:26:29,689 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/db2b44e044af4e31a347b7cc89501de6 2024-11-08T00:26:29,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/b79df6fe14f74e7d95471dc2bbdad9a6 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/b79df6fe14f74e7d95471dc2bbdad9a6 2024-11-08T00:26:29,697 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/b79df6fe14f74e7d95471dc2bbdad9a6, entries=150, sequenceid=339, filesize=30.5 K 2024-11-08T00:26:29,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/ff1156af08a5411b890669705e4326d5 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/ff1156af08a5411b890669705e4326d5 2024-11-08T00:26:29,701 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/ff1156af08a5411b890669705e4326d5, entries=150, sequenceid=339, filesize=12.0 K 2024-11-08T00:26:29,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/db2b44e044af4e31a347b7cc89501de6 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/db2b44e044af4e31a347b7cc89501de6 2024-11-08T00:26:29,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,706 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/db2b44e044af4e31a347b7cc89501de6, entries=150, sequenceid=339, filesize=12.0 K 2024-11-08T00:26:29,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,707 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 22b286596054a5e4265c6f73c6acf0d1 in 457ms, sequenceid=339, compaction requested=true 2024-11-08T00:26:29,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 22b286596054a5e4265c6f73c6acf0d1: 2024-11-08T00:26:29,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22b286596054a5e4265c6f73c6acf0d1:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T00:26:29,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:26:29,707 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T00:26:29,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22b286596054a5e4265c6f73c6acf0d1:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T00:26:29,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:26:29,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22b286596054a5e4265c6f73c6acf0d1:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:26:29,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:26:29,707 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T00:26:29,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,708 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T00:26:29,708 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 125668 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T00:26:29,708 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): 22b286596054a5e4265c6f73c6acf0d1/A is initiating minor compaction (all files) 2024-11-08T00:26:29,708 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): 22b286596054a5e4265c6f73c6acf0d1/B is initiating minor compaction (all files) 2024-11-08T00:26:29,708 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22b286596054a5e4265c6f73c6acf0d1/B in TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:29,708 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22b286596054a5e4265c6f73c6acf0d1/A in TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:29,708 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/6a227cc8a85b406ea3667e692ab65bf7, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/fc671ddb22a145a6ba5929fc0eedc7a8, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/0edbab4084c9499598678c26cb72e2c0, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/b79df6fe14f74e7d95471dc2bbdad9a6] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp, totalSize=122.7 K 2024-11-08T00:26:29,708 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/1c8ff002dc2f4d6ea0605bc759da8f27, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/824b72f7100645299f15f31b8e6f1450, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/23a604109da64446ae3066710e77cb8c, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/ff1156af08a5411b890669705e4326d5] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp, totalSize=48.7 K 2024-11-08T00:26:29,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,708 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:29,708 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. files: [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/6a227cc8a85b406ea3667e692ab65bf7, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/fc671ddb22a145a6ba5929fc0eedc7a8, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/0edbab4084c9499598678c26cb72e2c0, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/b79df6fe14f74e7d95471dc2bbdad9a6] 2024-11-08T00:26:29,709 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c8ff002dc2f4d6ea0605bc759da8f27, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731025585032 2024-11-08T00:26:29,709 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a227cc8a85b406ea3667e692ab65bf7, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731025585032 2024-11-08T00:26:29,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,709 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 824b72f7100645299f15f31b8e6f1450, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731025585770 2024-11-08T00:26:29,709 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting fc671ddb22a145a6ba5929fc0eedc7a8, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731025585770 2024-11-08T00:26:29,709 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23a604109da64446ae3066710e77cb8c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731025586926 2024-11-08T00:26:29,709 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 0edbab4084c9499598678c26cb72e2c0, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731025586926 2024-11-08T00:26:29,709 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff1156af08a5411b890669705e4326d5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1731025588105 2024-11-08T00:26:29,710 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting b79df6fe14f74e7d95471dc2bbdad9a6, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1731025588105 2024-11-08T00:26:29,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,715 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=22b286596054a5e4265c6f73c6acf0d1] 2024-11-08T00:26:29,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,717 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110866d0308e53ab4656b321130a0f7803d8_22b286596054a5e4265c6f73c6acf0d1 store=[table=TestAcidGuarantees family=A region=22b286596054a5e4265c6f73c6acf0d1] 2024-11-08T00:26:29,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,718 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22b286596054a5e4265c6f73c6acf0d1#B#compaction#581 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:26:29,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,718 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/1cb82e6c9a024e8197550ba86c7b9bac is 50, key is test_row_0/B:col10/1731025588105/Put/seqid=0 2024-11-08T00:26:29,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,719 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110866d0308e53ab4656b321130a0f7803d8_22b286596054a5e4265c6f73c6acf0d1, store=[table=TestAcidGuarantees family=A region=22b286596054a5e4265c6f73c6acf0d1] 2024-11-08T00:26:29,719 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110866d0308e53ab4656b321130a0f7803d8_22b286596054a5e4265c6f73c6acf0d1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=22b286596054a5e4265c6f73c6acf0d1] 2024-11-08T00:26:29,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742518_1694 (size=13085) 2024-11-08T00:26:29,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742519_1695 (size=4469) 2024-11-08T00:26:29,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:29,892 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:26:29,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,892 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 22b286596054a5e4265c6f73c6acf0d1 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T00:26:29,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,899 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108f5d9b60140c04c44939a3d9c9252b98d_22b286596054a5e4265c6f73c6acf0d1 is 50, key is test_row_0/A:col10/1731025589892/Put/seqid=0 2024-11-08T00:26:29,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742520_1696 (size=22618) 2024-11-08T00:26:29,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,902 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,905 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108f5d9b60140c04c44939a3d9c9252b98d_22b286596054a5e4265c6f73c6acf0d1 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108f5d9b60140c04c44939a3d9c9252b98d_22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:29,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,906 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/abdabb118a2a4c8cbceb378c9f8924dd, store: [table=TestAcidGuarantees family=A region=22b286596054a5e4265c6f73c6acf0d1] 2024-11-08T00:26:29,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,906 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/abdabb118a2a4c8cbceb378c9f8924dd is 175, key is test_row_0/A:col10/1731025589892/Put/seqid=0 2024-11-08T00:26:29,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:29,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742521_1697 (size=66023) 2024-11-08T00:26:29,915 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=352, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/abdabb118a2a4c8cbceb378c9f8924dd 2024-11-08T00:26:29,921 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/4ef61c41d4f44741933da3240da4bc2f is 50, key is test_row_0/B:col10/1731025589892/Put/seqid=0 2024-11-08T00:26:29,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742522_1698 (size=12301) 2024-11-08T00:26:29,924 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/4ef61c41d4f44741933da3240da4bc2f 2024-11-08T00:26:29,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:29,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025649918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:29,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:29,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:29,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:29,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:29,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025649921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:29,926 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:29,927 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:29,927 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:29,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:29,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025649924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:29,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:29,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:29,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:29,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:29,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025649925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:29,930 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/ac29ad39024c4199b63050f849fc4d34 is 50, key is test_row_0/C:col10/1731025589892/Put/seqid=0 2024-11-08T00:26:29,930 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:29,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:29,930 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:29,930 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:29,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025649925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:29,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:29,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:29,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:29,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742523_1699 (size=12301) 2024-11-08T00:26:29,934 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/ac29ad39024c4199b63050f849fc4d34 2024-11-08T00:26:29,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/abdabb118a2a4c8cbceb378c9f8924dd as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/abdabb118a2a4c8cbceb378c9f8924dd 2024-11-08T00:26:29,942 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/abdabb118a2a4c8cbceb378c9f8924dd, entries=350, sequenceid=352, filesize=64.5 K 2024-11-08T00:26:29,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/4ef61c41d4f44741933da3240da4bc2f as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/4ef61c41d4f44741933da3240da4bc2f 2024-11-08T00:26:29,946 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/4ef61c41d4f44741933da3240da4bc2f, entries=150, sequenceid=352, filesize=12.0 K 2024-11-08T00:26:29,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/ac29ad39024c4199b63050f849fc4d34 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/ac29ad39024c4199b63050f849fc4d34 2024-11-08T00:26:29,949 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/ac29ad39024c4199b63050f849fc4d34, entries=150, sequenceid=352, filesize=12.0 K 2024-11-08T00:26:29,949 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 22b286596054a5e4265c6f73c6acf0d1 in 57ms, sequenceid=352, compaction requested=true 2024-11-08T00:26:29,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 22b286596054a5e4265c6f73c6acf0d1: 2024-11-08T00:26:29,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22b286596054a5e4265c6f73c6acf0d1:A, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:26:29,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-08T00:26:29,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22b286596054a5e4265c6f73c6acf0d1:B, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:26:29,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-08T00:26:29,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22b286596054a5e4265c6f73c6acf0d1:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:26:29,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-08T00:26:30,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:30,041 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:26:30,041 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 22b286596054a5e4265c6f73c6acf0d1 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-08T00:26:30,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108b90a64edc8c24e6e82a0c9058e2049b3_22b286596054a5e4265c6f73c6acf0d1 is 50, key is test_row_0/A:col10/1731025589921/Put/seqid=0 2024-11-08T00:26:30,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742524_1700 (size=17534) 2024-11-08T00:26:30,053 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:30,055 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108b90a64edc8c24e6e82a0c9058e2049b3_22b286596054a5e4265c6f73c6acf0d1 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108b90a64edc8c24e6e82a0c9058e2049b3_22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:30,056 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/4d768e9836064c0eba2c7b3866ad8fc7, store: [table=TestAcidGuarantees family=A region=22b286596054a5e4265c6f73c6acf0d1] 2024-11-08T00:26:30,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:30,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025650050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:30,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:30,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025650050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:30,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:30,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/4d768e9836064c0eba2c7b3866ad8fc7 is 175, key is test_row_0/A:col10/1731025589921/Put/seqid=0 2024-11-08T00:26:30,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025650051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:30,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:30,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025650052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:30,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:30,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025650052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:30,057 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:30,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:30,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:30,057 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:30,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:30,057 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:30,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:30,058 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:30,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:30,058 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:30,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:30,058 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:30,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:30,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:30,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:30,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742525_1701 (size=48639) 2024-11-08T00:26:30,132 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22b286596054a5e4265c6f73c6acf0d1#A#compaction#580 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:26:30,132 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/440f4b47c3d64f738eca8e80419088a6 is 175, key is test_row_0/A:col10/1731025588105/Put/seqid=0 2024-11-08T00:26:30,134 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/1cb82e6c9a024e8197550ba86c7b9bac as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/1cb82e6c9a024e8197550ba86c7b9bac 2024-11-08T00:26:30,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742526_1702 (size=32039) 2024-11-08T00:26:30,139 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 22b286596054a5e4265c6f73c6acf0d1/B of 22b286596054a5e4265c6f73c6acf0d1 into 1cb82e6c9a024e8197550ba86c7b9bac(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:26:30,139 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22b286596054a5e4265c6f73c6acf0d1: 2024-11-08T00:26:30,139 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., storeName=22b286596054a5e4265c6f73c6acf0d1/B, priority=12, startTime=1731025589707; duration=0sec 2024-11-08T00:26:30,139 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-08T00:26:30,139 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22b286596054a5e4265c6f73c6acf0d1:B 2024-11-08T00:26:30,139 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 4 compacting, 1 eligible, 16 blocking 2024-11-08T00:26:30,139 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-08T00:26:30,139 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-08T00:26:30,139 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. because compaction request was cancelled 2024-11-08T00:26:30,139 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22b286596054a5e4265c6f73c6acf0d1:A 2024-11-08T00:26:30,140 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-08T00:26:30,140 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62153 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-08T00:26:30,140 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): 22b286596054a5e4265c6f73c6acf0d1/C is initiating minor compaction (all files) 2024-11-08T00:26:30,140 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22b286596054a5e4265c6f73c6acf0d1/C in TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:30,140 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/52b8c034c60c4de783fca133084ce0bd, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/03a46c21944a4b56af542ea48a43ced3, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/368e2e2b21154bc491d609bd195f55d7, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/db2b44e044af4e31a347b7cc89501de6, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/ac29ad39024c4199b63050f849fc4d34] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp, totalSize=60.7 K 2024-11-08T00:26:30,140 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52b8c034c60c4de783fca133084ce0bd, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731025585032 2024-11-08T00:26:30,141 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03a46c21944a4b56af542ea48a43ced3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731025585770 2024-11-08T00:26:30,141 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 368e2e2b21154bc491d609bd195f55d7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731025586926 2024-11-08T00:26:30,141 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting db2b44e044af4e31a347b7cc89501de6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1731025588105 2024-11-08T00:26:30,141 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac29ad39024c4199b63050f849fc4d34, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731025589251 2024-11-08T00:26:30,148 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22b286596054a5e4265c6f73c6acf0d1#C#compaction#586 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:26:30,148 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/d64221850c75472ea7232b9e04aaefa8 is 50, key is test_row_0/C:col10/1731025589892/Put/seqid=0 2024-11-08T00:26:30,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742527_1703 (size=13119) 2024-11-08T00:26:30,156 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/d64221850c75472ea7232b9e04aaefa8 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/d64221850c75472ea7232b9e04aaefa8 2024-11-08T00:26:30,159 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 22b286596054a5e4265c6f73c6acf0d1/C of 22b286596054a5e4265c6f73c6acf0d1 into d64221850c75472ea7232b9e04aaefa8(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:26:30,159 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22b286596054a5e4265c6f73c6acf0d1: 2024-11-08T00:26:30,159 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., storeName=22b286596054a5e4265c6f73c6acf0d1/C, priority=11, startTime=1731025589951; duration=0sec 2024-11-08T00:26:30,159 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-08T00:26:30,159 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22b286596054a5e4265c6f73c6acf0d1:C 2024-11-08T00:26:30,159 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22b286596054a5e4265c6f73c6acf0d1:B 2024-11-08T00:26:30,159 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22b286596054a5e4265c6f73c6acf0d1:C 2024-11-08T00:26:30,160 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-08T00:26:30,160 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-08T00:26:30,160 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-08T00:26:30,160 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. because compaction request was cancelled 2024-11-08T00:26:30,160 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22b286596054a5e4265c6f73c6acf0d1:B 2024-11-08T00:26:30,160 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-08T00:26:30,160 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-08T00:26:30,160 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-08T00:26:30,160 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. because compaction request was cancelled 2024-11-08T00:26:30,160 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22b286596054a5e4265c6f73c6acf0d1:C 2024-11-08T00:26:30,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:30,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025650167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:30,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:30,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:30,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:30,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:30,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025650167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:30,172 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:30,172 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:30,172 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:30,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:30,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025650167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:30,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:30,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:30,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:30,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:30,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025650167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:30,173 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:30,173 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:30,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:30,173 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:30,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025650167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:30,173 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:30,173 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:30,173 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:30,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:30,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025650377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:30,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:30,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:30,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025650377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:30,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:30,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:30,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:30,380 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:30,380 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:30,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025650377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:30,380 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:30,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:30,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025650377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:30,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:30,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:30,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:30,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:30,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:30,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:30,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:30,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025650378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:30,381 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:30,381 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:30,381 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:30,460 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=377, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/4d768e9836064c0eba2c7b3866ad8fc7 2024-11-08T00:26:30,465 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/07778fe7495c4a23a651120ab5195f9b is 50, key is test_row_0/B:col10/1731025589921/Put/seqid=0 2024-11-08T00:26:30,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742528_1704 (size=12301) 2024-11-08T00:26:30,539 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/440f4b47c3d64f738eca8e80419088a6 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/440f4b47c3d64f738eca8e80419088a6 2024-11-08T00:26:30,542 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 22b286596054a5e4265c6f73c6acf0d1/A of 22b286596054a5e4265c6f73c6acf0d1 into 440f4b47c3d64f738eca8e80419088a6(size=31.3 K), total size for store is 95.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:26:30,542 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22b286596054a5e4265c6f73c6acf0d1: 2024-11-08T00:26:30,543 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., storeName=22b286596054a5e4265c6f73c6acf0d1/A, priority=12, startTime=1731025589707; duration=0sec 2024-11-08T00:26:30,543 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:26:30,543 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22b286596054a5e4265c6f73c6acf0d1:A 2024-11-08T00:26:30,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:30,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025650687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:30,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:30,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:30,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025650687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:30,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:30,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:30,688 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:30,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:30,688 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:30,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:30,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025650687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:30,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:30,689 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:30,689 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:30,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025650687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:30,689 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:30,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:30,689 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:30,689 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:30,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025650687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:30,689 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:30,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:30,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:30,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:30,869 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/07778fe7495c4a23a651120ab5195f9b 2024-11-08T00:26:30,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/7fb91777fa024d298168e256720e44c7 is 50, key is test_row_0/C:col10/1731025589921/Put/seqid=0 2024-11-08T00:26:30,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742529_1705 (size=12301) 2024-11-08T00:26:31,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-08T00:26:31,057 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:26:31,057 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-08T00:26:31,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-08T00:26:31,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-08T00:26:31,059 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T00:26:31,059 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T00:26:31,060 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T00:26:31,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-08T00:26:31,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:31,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025651197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:31,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:31,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025651197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:31,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:31,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:31,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:31,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:31,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:31,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025651198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:31,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025651198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:31,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:31,200 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:31,200 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:31,200 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:31,200 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:31,200 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:31,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025651198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:31,200 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:31,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:31,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:31,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:31,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:31,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:31,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:31,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-08T00:26:31,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:31,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. as already flushing 2024-11-08T00:26:31,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:31,212 ERROR [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:31,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:31,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T00:26:31,282 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/7fb91777fa024d298168e256720e44c7 2024-11-08T00:26:31,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/4d768e9836064c0eba2c7b3866ad8fc7 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/4d768e9836064c0eba2c7b3866ad8fc7 2024-11-08T00:26:31,287 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/4d768e9836064c0eba2c7b3866ad8fc7, entries=250, sequenceid=377, filesize=47.5 K 2024-11-08T00:26:31,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/07778fe7495c4a23a651120ab5195f9b as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/07778fe7495c4a23a651120ab5195f9b 2024-11-08T00:26:31,290 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/07778fe7495c4a23a651120ab5195f9b, entries=150, sequenceid=377, filesize=12.0 K 2024-11-08T00:26:31,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/7fb91777fa024d298168e256720e44c7 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/7fb91777fa024d298168e256720e44c7 2024-11-08T00:26:31,293 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/7fb91777fa024d298168e256720e44c7, entries=150, sequenceid=377, filesize=12.0 K 2024-11-08T00:26:31,293 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 22b286596054a5e4265c6f73c6acf0d1 in 1252ms, sequenceid=377, compaction requested=true 2024-11-08T00:26:31,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 22b286596054a5e4265c6f73c6acf0d1: 2024-11-08T00:26:31,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22b286596054a5e4265c6f73c6acf0d1:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T00:26:31,294 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:26:31,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:26:31,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22b286596054a5e4265c6f73c6acf0d1:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T00:26:31,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:26:31,294 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:26:31,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22b286596054a5e4265c6f73c6acf0d1:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:26:31,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:26:31,294 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:26:31,294 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 146701 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:26:31,294 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): 22b286596054a5e4265c6f73c6acf0d1/B is initiating minor compaction (all files) 2024-11-08T00:26:31,294 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): 22b286596054a5e4265c6f73c6acf0d1/A is initiating minor compaction (all files) 2024-11-08T00:26:31,294 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22b286596054a5e4265c6f73c6acf0d1/A in TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:31,294 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22b286596054a5e4265c6f73c6acf0d1/B in TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:31,294 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/1cb82e6c9a024e8197550ba86c7b9bac, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/4ef61c41d4f44741933da3240da4bc2f, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/07778fe7495c4a23a651120ab5195f9b] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp, totalSize=36.8 K 2024-11-08T00:26:31,294 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/440f4b47c3d64f738eca8e80419088a6, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/abdabb118a2a4c8cbceb378c9f8924dd, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/4d768e9836064c0eba2c7b3866ad8fc7] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp, totalSize=143.3 K 2024-11-08T00:26:31,294 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:31,294 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. files: [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/440f4b47c3d64f738eca8e80419088a6, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/abdabb118a2a4c8cbceb378c9f8924dd, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/4d768e9836064c0eba2c7b3866ad8fc7] 2024-11-08T00:26:31,295 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 1cb82e6c9a024e8197550ba86c7b9bac, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1731025588105 2024-11-08T00:26:31,295 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 440f4b47c3d64f738eca8e80419088a6, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1731025588105 2024-11-08T00:26:31,295 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ef61c41d4f44741933da3240da4bc2f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731025589251 2024-11-08T00:26:31,295 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting abdabb118a2a4c8cbceb378c9f8924dd, keycount=350, bloomtype=ROW, size=64.5 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731025589251 2024-11-08T00:26:31,295 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 07778fe7495c4a23a651120ab5195f9b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1731025589921 2024-11-08T00:26:31,295 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d768e9836064c0eba2c7b3866ad8fc7, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1731025589914 2024-11-08T00:26:31,299 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=22b286596054a5e4265c6f73c6acf0d1] 2024-11-08T00:26:31,301 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22b286596054a5e4265c6f73c6acf0d1#B#compaction#589 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:26:31,301 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/f98494c77740482eb2e2be5975f27502 is 50, key is test_row_0/B:col10/1731025589921/Put/seqid=0 2024-11-08T00:26:31,301 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241108a4a14a1c689e45419c6c9546dd486fc2_22b286596054a5e4265c6f73c6acf0d1 store=[table=TestAcidGuarantees family=A region=22b286596054a5e4265c6f73c6acf0d1] 2024-11-08T00:26:31,303 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241108a4a14a1c689e45419c6c9546dd486fc2_22b286596054a5e4265c6f73c6acf0d1, store=[table=TestAcidGuarantees family=A region=22b286596054a5e4265c6f73c6acf0d1] 2024-11-08T00:26:31,303 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108a4a14a1c689e45419c6c9546dd486fc2_22b286596054a5e4265c6f73c6acf0d1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=22b286596054a5e4265c6f73c6acf0d1] 2024-11-08T00:26:31,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742531_1707 (size=4469) 2024-11-08T00:26:31,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742530_1706 (size=13187) 2024-11-08T00:26:31,307 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22b286596054a5e4265c6f73c6acf0d1#A#compaction#590 average throughput is 3.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:26:31,308 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/00f033d8aba1401ca73b89d5981b7427 is 175, key is test_row_0/A:col10/1731025589921/Put/seqid=0 2024-11-08T00:26:31,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742532_1708 (size=32141) 2024-11-08T00:26:31,364 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40479 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-08T00:26:31,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:31,364 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing 22b286596054a5e4265c6f73c6acf0d1 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T00:26:31,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108ad611dc453d44a8d8d6caf13202e6376_22b286596054a5e4265c6f73c6acf0d1 is 50, key is test_row_0/A:col10/1731025590050/Put/seqid=0 2024-11-08T00:26:31,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742533_1709 (size=12454) 2024-11-08T00:26:31,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-08T00:26:31,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-08T00:26:31,710 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/f98494c77740482eb2e2be5975f27502 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/f98494c77740482eb2e2be5975f27502 2024-11-08T00:26:31,714 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/00f033d8aba1401ca73b89d5981b7427 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/00f033d8aba1401ca73b89d5981b7427 2024-11-08T00:26:31,714 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22b286596054a5e4265c6f73c6acf0d1/B of 22b286596054a5e4265c6f73c6acf0d1 into f98494c77740482eb2e2be5975f27502(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:26:31,714 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22b286596054a5e4265c6f73c6acf0d1: 2024-11-08T00:26:31,714 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., storeName=22b286596054a5e4265c6f73c6acf0d1/B, priority=13, startTime=1731025591294; duration=0sec 2024-11-08T00:26:31,714 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:26:31,714 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22b286596054a5e4265c6f73c6acf0d1:B 2024-11-08T00:26:31,714 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-08T00:26:31,715 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-08T00:26:31,715 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-08T00:26:31,715 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. because compaction request was cancelled 2024-11-08T00:26:31,715 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22b286596054a5e4265c6f73c6acf0d1:C 2024-11-08T00:26:31,717 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22b286596054a5e4265c6f73c6acf0d1/A of 22b286596054a5e4265c6f73c6acf0d1 into 00f033d8aba1401ca73b89d5981b7427(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:26:31,717 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22b286596054a5e4265c6f73c6acf0d1: 2024-11-08T00:26:31,717 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., storeName=22b286596054a5e4265c6f73c6acf0d1/A, priority=13, startTime=1731025591293; duration=0sec 2024-11-08T00:26:31,717 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:26:31,717 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22b286596054a5e4265c6f73c6acf0d1:A 2024-11-08T00:26:31,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:31,778 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108ad611dc453d44a8d8d6caf13202e6376_22b286596054a5e4265c6f73c6acf0d1 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108ad611dc453d44a8d8d6caf13202e6376_22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:31,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/5cf77d09123c4da7b3d398b043065f89, store: [table=TestAcidGuarantees family=A region=22b286596054a5e4265c6f73c6acf0d1] 2024-11-08T00:26:31,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/5cf77d09123c4da7b3d398b043065f89 is 175, key is test_row_0/A:col10/1731025590050/Put/seqid=0 2024-11-08T00:26:31,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742534_1710 (size=31255) 2024-11-08T00:26:31,783 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=391, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/5cf77d09123c4da7b3d398b043065f89 2024-11-08T00:26:31,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/f50b6009227945d095a75d0262983a84 is 50, key is test_row_0/B:col10/1731025590050/Put/seqid=0 2024-11-08T00:26:31,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742535_1711 (size=12301) 2024-11-08T00:26:32,191 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/f50b6009227945d095a75d0262983a84 2024-11-08T00:26:32,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/81f7a1f6da8b48279f1816cb92e4f5be is 50, key is test_row_0/C:col10/1731025590050/Put/seqid=0 2024-11-08T00:26:32,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-08T00:26:32,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742536_1712 (size=12301) 2024-11-08T00:26:32,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:32,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. as already flushing 2024-11-08T00:26:32,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:32,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025652229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:32,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:32,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025652230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:32,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:32,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:32,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:32,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:32,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:32,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:32,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:32,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025652231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:32,234 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:32,234 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:32,234 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:32,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:32,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025652231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:32,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:32,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:32,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:32,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:32,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025652232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:32,235 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:32,235 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:32,235 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:32,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:32,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025652337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:32,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:32,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025652337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:32,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:32,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:32,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:32,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:32,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:32,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:32,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025652338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:32,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:32,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:32,339 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:32,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025652338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:32,339 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:32,339 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:32,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:32,339 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:32,339 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:32,339 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:32,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025652338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:32,339 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:32,339 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:32,339 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:32,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:32,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025652547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:32,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:32,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:32,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:32,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:32,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025652547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:32,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:32,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025652548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:32,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:32,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:32,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:32,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:32,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:32,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:32,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:32,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025652548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:32,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:32,550 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:32,550 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:32,550 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:32,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025652548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:32,550 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:32,550 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:32,550 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:32,600 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/81f7a1f6da8b48279f1816cb92e4f5be 2024-11-08T00:26:32,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/5cf77d09123c4da7b3d398b043065f89 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/5cf77d09123c4da7b3d398b043065f89 2024-11-08T00:26:32,605 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/5cf77d09123c4da7b3d398b043065f89, entries=150, sequenceid=391, filesize=30.5 K 2024-11-08T00:26:32,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/f50b6009227945d095a75d0262983a84 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/f50b6009227945d095a75d0262983a84 2024-11-08T00:26:32,612 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/f50b6009227945d095a75d0262983a84, entries=150, sequenceid=391, filesize=12.0 K 2024-11-08T00:26:32,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/81f7a1f6da8b48279f1816cb92e4f5be as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/81f7a1f6da8b48279f1816cb92e4f5be 2024-11-08T00:26:32,615 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/81f7a1f6da8b48279f1816cb92e4f5be, entries=150, sequenceid=391, filesize=12.0 K 2024-11-08T00:26:32,615 INFO [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 22b286596054a5e4265c6f73c6acf0d1 in 1251ms, sequenceid=391, compaction requested=true 2024-11-08T00:26:32,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for 22b286596054a5e4265c6f73c6acf0d1: 2024-11-08T00:26:32,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:32,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/3302f0f507bd:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-08T00:26:32,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-08T00:26:32,618 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-08T00:26:32,618 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5570 sec 2024-11-08T00:26:32,619 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 1.5600 sec 2024-11-08T00:26:32,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:32,860 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:26:32,860 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 22b286596054a5e4265c6f73c6acf0d1 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-08T00:26:32,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108fd88b2773fcf44eabdabb625caee29b3_22b286596054a5e4265c6f73c6acf0d1 is 50, key is test_row_0/A:col10/1731025592859/Put/seqid=0 2024-11-08T00:26:32,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:32,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025652862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:32,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:32,867 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:32,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025652862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:32,867 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:32,867 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:32,867 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:32,867 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:32,867 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:32,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742537_1713 (size=12454) 2024-11-08T00:26:32,869 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:32,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:32,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:32,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025652866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:32,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025652866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:32,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:32,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:32,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025652866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:32,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:32,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:32,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:32,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:32,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:32,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:32,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:32,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:32,873 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108fd88b2773fcf44eabdabb625caee29b3_22b286596054a5e4265c6f73c6acf0d1 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108fd88b2773fcf44eabdabb625caee29b3_22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:32,873 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/ff6d482b91304236a471afc439bb4dd8, store: [table=TestAcidGuarantees family=A region=22b286596054a5e4265c6f73c6acf0d1] 2024-11-08T00:26:32,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/ff6d482b91304236a471afc439bb4dd8 is 175, key is test_row_0/A:col10/1731025592859/Put/seqid=0 2024-11-08T00:26:32,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742538_1714 (size=31255) 2024-11-08T00:26:32,919 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T00:26:32,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:32,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025652977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:32,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:32,979 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:32,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025652977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:32,979 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:32,979 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:32,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:32,979 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:32,979 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:32,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025652978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:32,979 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:32,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:32,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:32,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025652978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:32,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:32,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:32,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:32,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025652978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:32,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:32,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:32,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:32,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:32,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:32,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:33,122 INFO [Thread-2672 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2672. 2024-11-08T00:26:33,122 DEBUG [Thread-2672 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T00:26:33,122 DEBUG [Thread-2672 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:26:33,122 DEBUG [Thread-2672 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:26:33,122 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:26:33,124 INFO [Thread-2666 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2666. 2024-11-08T00:26:33,124 INFO [Thread-2670 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2670. 2024-11-08T00:26:33,124 DEBUG [Thread-2666 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T00:26:33,124 DEBUG [Thread-2666 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:26:33,124 DEBUG [Thread-2670 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T00:26:33,124 DEBUG [Thread-2670 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:26:33,124 DEBUG [Thread-2666 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:26:33,124 DEBUG [Thread-2670 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:26:33,124 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:26:33,124 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:26:33,125 INFO [Thread-2674 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2674. 2024-11-08T00:26:33,125 DEBUG [Thread-2674 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T00:26:33,125 DEBUG [Thread-2674 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:26:33,125 DEBUG [Thread-2674 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:26:33,125 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:26:33,127 INFO [Thread-2668 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2668. 2024-11-08T00:26:33,127 DEBUG [Thread-2668 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T00:26:33,127 DEBUG [Thread-2668 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:26:33,127 DEBUG [Thread-2668 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:26:33,127 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:26:33,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:33,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:33,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:33,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025653187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:33,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025653187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:33,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025653188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:33,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:33,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:33,189 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:33,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:33,189 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:33,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025653188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:33,189 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:33,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:33,189 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:33,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025653188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:33,189 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:33,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:33,189 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:33,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:33,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:33,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:33,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:33,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:33,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:33,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-08T00:26:33,208 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:26:33,278 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=420, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/ff6d482b91304236a471afc439bb4dd8 2024-11-08T00:26:33,291 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/c9e38f52b4554cd68e27c92d4617a778 is 50, key is test_row_0/B:col10/1731025592859/Put/seqid=0 2024-11-08T00:26:33,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742539_1715 (size=12301) 2024-11-08T00:26:33,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:33,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025653498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:33,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:33,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025653499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:33,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:33,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:33,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025653499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:33,501 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:33,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025653500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:33,501 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:33,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:33,501 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:33,501 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:33,501 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:33,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025653500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:33,501 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:33,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:33,501 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:33,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:33,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:33,502 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:33,502 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:33,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:33,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:33,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:33,695 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/c9e38f52b4554cd68e27c92d4617a778 2024-11-08T00:26:33,700 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/b4e564b1feaa436083684cfb8bb8bd93 is 50, key is test_row_0/C:col10/1731025592859/Put/seqid=0 2024-11-08T00:26:33,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742540_1716 (size=12301) 2024-11-08T00:26:34,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:34,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:34,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46052 deadline: 1731025654008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:34,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45986 deadline: 1731025654009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:34,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:34,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:34,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45976 deadline: 1731025654009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:34,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:34,011 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:34,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:34,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-08T00:26:34,011 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:34,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45996 deadline: 1731025654010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:34,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:34,011 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:34,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46048 deadline: 1731025654010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:34,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:34,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:34,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:34,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:34,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:34,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:34,012 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 , the old value is region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:34,012 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T00:26:34,012 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., hostname=3302f0f507bd,40479,1731025415856, seqNum=5 because the exception is null or not the one we care about 2024-11-08T00:26:34,105 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/b4e564b1feaa436083684cfb8bb8bd93 2024-11-08T00:26:34,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/ff6d482b91304236a471afc439bb4dd8 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/ff6d482b91304236a471afc439bb4dd8 2024-11-08T00:26:34,121 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/ff6d482b91304236a471afc439bb4dd8, entries=150, sequenceid=420, filesize=30.5 K 2024-11-08T00:26:34,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/c9e38f52b4554cd68e27c92d4617a778 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/c9e38f52b4554cd68e27c92d4617a778 2024-11-08T00:26:34,128 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/c9e38f52b4554cd68e27c92d4617a778, entries=150, sequenceid=420, filesize=12.0 K 2024-11-08T00:26:34,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/b4e564b1feaa436083684cfb8bb8bd93 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/b4e564b1feaa436083684cfb8bb8bd93 2024-11-08T00:26:34,132 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/b4e564b1feaa436083684cfb8bb8bd93, entries=150, sequenceid=420, filesize=12.0 K 2024-11-08T00:26:34,133 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 22b286596054a5e4265c6f73c6acf0d1 in 1273ms, sequenceid=420, compaction requested=true 2024-11-08T00:26:34,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 22b286596054a5e4265c6f73c6acf0d1: 2024-11-08T00:26:34,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22b286596054a5e4265c6f73c6acf0d1:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T00:26:34,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:26:34,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22b286596054a5e4265c6f73c6acf0d1:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T00:26:34,133 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:26:34,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:26:34,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 22b286596054a5e4265c6f73c6acf0d1:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T00:26:34,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:26:34,133 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T00:26:34,134 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:26:34,134 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94651 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T00:26:34,134 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1541): 22b286596054a5e4265c6f73c6acf0d1/A is initiating minor compaction (all files) 2024-11-08T00:26:34,134 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): 22b286596054a5e4265c6f73c6acf0d1/B is initiating minor compaction (all files) 2024-11-08T00:26:34,134 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22b286596054a5e4265c6f73c6acf0d1/A in TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:34,134 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22b286596054a5e4265c6f73c6acf0d1/B in TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:34,134 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/f98494c77740482eb2e2be5975f27502, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/f50b6009227945d095a75d0262983a84, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/c9e38f52b4554cd68e27c92d4617a778] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp, totalSize=36.9 K 2024-11-08T00:26:34,134 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/00f033d8aba1401ca73b89d5981b7427, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/5cf77d09123c4da7b3d398b043065f89, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/ff6d482b91304236a471afc439bb4dd8] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp, totalSize=92.4 K 2024-11-08T00:26:34,134 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:34,134 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. files: [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/00f033d8aba1401ca73b89d5981b7427, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/5cf77d09123c4da7b3d398b043065f89, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/ff6d482b91304236a471afc439bb4dd8] 2024-11-08T00:26:34,134 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting f98494c77740482eb2e2be5975f27502, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1731025589921 2024-11-08T00:26:34,135 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00f033d8aba1401ca73b89d5981b7427, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1731025589921 2024-11-08T00:26:34,135 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting f50b6009227945d095a75d0262983a84, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1731025590048 2024-11-08T00:26:34,135 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5cf77d09123c4da7b3d398b043065f89, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1731025590048 2024-11-08T00:26:34,135 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting c9e38f52b4554cd68e27c92d4617a778, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1731025592230 2024-11-08T00:26:34,135 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff6d482b91304236a471afc439bb4dd8, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1731025592230 2024-11-08T00:26:34,140 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=22b286596054a5e4265c6f73c6acf0d1] 2024-11-08T00:26:34,142 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22b286596054a5e4265c6f73c6acf0d1#B#compaction#597 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:26:34,142 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/e8426c72b16e4f1e847d795796eb191e is 50, key is test_row_0/B:col10/1731025592859/Put/seqid=0 2024-11-08T00:26:34,142 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241108f92c8b70e2d0460fa6c817e7dbc96c8a_22b286596054a5e4265c6f73c6acf0d1 store=[table=TestAcidGuarantees family=A region=22b286596054a5e4265c6f73c6acf0d1] 2024-11-08T00:26:34,144 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241108f92c8b70e2d0460fa6c817e7dbc96c8a_22b286596054a5e4265c6f73c6acf0d1, store=[table=TestAcidGuarantees family=A region=22b286596054a5e4265c6f73c6acf0d1] 2024-11-08T00:26:34,145 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108f92c8b70e2d0460fa6c817e7dbc96c8a_22b286596054a5e4265c6f73c6acf0d1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=22b286596054a5e4265c6f73c6acf0d1] 2024-11-08T00:26:34,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742541_1717 (size=13289) 2024-11-08T00:26:34,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742542_1718 (size=4469) 2024-11-08T00:26:34,149 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22b286596054a5e4265c6f73c6acf0d1#A#compaction#598 average throughput is 2.71 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:26:34,150 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/3f907a9d688d49b993654ea4daa78891 is 175, key is test_row_0/A:col10/1731025592859/Put/seqid=0 2024-11-08T00:26:34,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742543_1719 (size=32243) 2024-11-08T00:26:34,557 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/e8426c72b16e4f1e847d795796eb191e as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/e8426c72b16e4f1e847d795796eb191e 2024-11-08T00:26:34,560 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/3f907a9d688d49b993654ea4daa78891 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/3f907a9d688d49b993654ea4daa78891 2024-11-08T00:26:34,562 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22b286596054a5e4265c6f73c6acf0d1/B of 22b286596054a5e4265c6f73c6acf0d1 into e8426c72b16e4f1e847d795796eb191e(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:26:34,562 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22b286596054a5e4265c6f73c6acf0d1: 2024-11-08T00:26:34,563 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., storeName=22b286596054a5e4265c6f73c6acf0d1/B, priority=13, startTime=1731025594133; duration=0sec 2024-11-08T00:26:34,563 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T00:26:34,563 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22b286596054a5e4265c6f73c6acf0d1:B 2024-11-08T00:26:34,563 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T00:26:34,564 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T00:26:34,564 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1541): 22b286596054a5e4265c6f73c6acf0d1/C is initiating minor compaction (all files) 2024-11-08T00:26:34,564 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 22b286596054a5e4265c6f73c6acf0d1/C in TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:34,564 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/d64221850c75472ea7232b9e04aaefa8, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/7fb91777fa024d298168e256720e44c7, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/81f7a1f6da8b48279f1816cb92e4f5be, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/b4e564b1feaa436083684cfb8bb8bd93] into tmpdir=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp, totalSize=48.8 K 2024-11-08T00:26:34,565 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting d64221850c75472ea7232b9e04aaefa8, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731025589251 2024-11-08T00:26:34,565 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 22b286596054a5e4265c6f73c6acf0d1/A of 22b286596054a5e4265c6f73c6acf0d1 into 3f907a9d688d49b993654ea4daa78891(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:26:34,565 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22b286596054a5e4265c6f73c6acf0d1: 2024-11-08T00:26:34,565 INFO [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., storeName=22b286596054a5e4265c6f73c6acf0d1/A, priority=13, startTime=1731025594133; duration=0sec 2024-11-08T00:26:34,565 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:26:34,565 DEBUG [RS:0;3302f0f507bd:40479-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22b286596054a5e4265c6f73c6acf0d1:A 2024-11-08T00:26:34,565 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fb91777fa024d298168e256720e44c7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1731025589921 2024-11-08T00:26:34,565 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting 81f7a1f6da8b48279f1816cb92e4f5be, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1731025590048 2024-11-08T00:26:34,565 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] compactions.Compactor(225): Compacting b4e564b1feaa436083684cfb8bb8bd93, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=420, earliestPutTs=1731025592230 2024-11-08T00:26:34,572 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 22b286596054a5e4265c6f73c6acf0d1#C#compaction#599 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T00:26:34,572 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/89588cade07d486a95e1695b8dad3b74 is 50, key is test_row_0/C:col10/1731025592859/Put/seqid=0 2024-11-08T00:26:34,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742544_1720 (size=13255) 2024-11-08T00:26:34,985 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/89588cade07d486a95e1695b8dad3b74 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/89588cade07d486a95e1695b8dad3b74 2024-11-08T00:26:34,992 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 22b286596054a5e4265c6f73c6acf0d1/C of 22b286596054a5e4265c6f73c6acf0d1 into 89588cade07d486a95e1695b8dad3b74(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T00:26:34,992 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 22b286596054a5e4265c6f73c6acf0d1: 2024-11-08T00:26:34,992 INFO [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1., storeName=22b286596054a5e4265c6f73c6acf0d1/C, priority=12, startTime=1731025594133; duration=0sec 2024-11-08T00:26:34,992 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T00:26:34,993 DEBUG [RS:0;3302f0f507bd:40479-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 22b286596054a5e4265c6f73c6acf0d1:C 2024-11-08T00:26:35,020 INFO [Thread-2655 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2655. 2024-11-08T00:26:35,020 DEBUG [Thread-2655 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T00:26:35,020 DEBUG [Thread-2655 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:26:35,020 DEBUG [Thread-2655 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:26:35,021 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:26:35,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40479 {}] regionserver.HRegion(8855): Flush requested on 22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:35,031 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T00:26:35,031 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 22b286596054a5e4265c6f73c6acf0d1 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-08T00:26:35,031 INFO [Thread-2663 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2663. 2024-11-08T00:26:35,031 INFO [Thread-2661 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2661. 2024-11-08T00:26:35,031 INFO [Thread-2657 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2657. 2024-11-08T00:26:35,031 DEBUG [Thread-2663 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T00:26:35,031 DEBUG [Thread-2661 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T00:26:35,031 DEBUG [Thread-2657 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T00:26:35,031 DEBUG [Thread-2663 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:26:35,031 DEBUG [Thread-2661 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:26:35,031 DEBUG [Thread-2657 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:26:35,031 DEBUG [Thread-2663 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:26:35,031 DEBUG [Thread-2657 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:26:35,031 DEBUG [Thread-2661 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:26:35,031 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:26:35,032 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:26:35,032 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:26:35,033 INFO [Thread-2659 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2659. 2024-11-08T00:26:35,033 DEBUG [Thread-2659 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T00:26:35,033 DEBUG [Thread-2659 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:26:35,033 DEBUG [Thread-2659 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:26:35,033 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:26:35,034 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-08T00:26:35,034 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-11-08T00:26:35,034 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-08T00:26:35,034 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-11-08T00:26:35,034 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-11-08T00:26:35,034 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-08T00:26:35,034 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-08T00:26:35,034 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3695 2024-11-08T00:26:35,034 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3793 2024-11-08T00:26:35,034 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3558 2024-11-08T00:26:35,034 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3707 2024-11-08T00:26:35,034 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3791 2024-11-08T00:26:35,034 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-08T00:26:35,034 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T00:26:35,034 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T00:26:35,034 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:26:35,034 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:26:35,034 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:26:35,035 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-08T00:26:35,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-08T00:26:35,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-08T00:26:35,039 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731025595039"}]},"ts":"1731025595039"} 2024-11-08T00:26:35,040 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411085f29f7a8392a47cfaa57fca01bde52bc_22b286596054a5e4265c6f73c6acf0d1 is 50, key is test_row_0/A:col10/1731025595028/Put/seqid=0 2024-11-08T00:26:35,041 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-08T00:26:35,041 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-08T00:26:35,041 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-08T00:26:35,043 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=168, ppid=167, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=22b286596054a5e4265c6f73c6acf0d1, UNASSIGN}] 2024-11-08T00:26:35,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742545_1721 (size=12454) 2024-11-08T00:26:35,044 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=168, ppid=167, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=22b286596054a5e4265c6f73c6acf0d1, UNASSIGN 2024-11-08T00:26:35,045 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=168 updating hbase:meta row=22b286596054a5e4265c6f73c6acf0d1, regionState=CLOSING, regionLocation=3302f0f507bd,40479,1731025415856 2024-11-08T00:26:35,046 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=168, ppid=167, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=22b286596054a5e4265c6f73c6acf0d1, UNASSIGN because future has completed 2024-11-08T00:26:35,046 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-08T00:26:35,046 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; CloseRegionProcedure 22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856}] 2024-11-08T00:26:35,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-08T00:26:35,199 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] handler.UnassignRegionHandler(122): Close 22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:35,199 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-08T00:26:35,199 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1722): Closing 22b286596054a5e4265c6f73c6acf0d1, disabling compactions & flushes 2024-11-08T00:26:35,199 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:35,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-08T00:26:35,445 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T00:26:35,453 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411085f29f7a8392a47cfaa57fca01bde52bc_22b286596054a5e4265c6f73c6acf0d1 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411085f29f7a8392a47cfaa57fca01bde52bc_22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:35,455 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/1d77141220eb462d9bb2d82e6566176b, store: [table=TestAcidGuarantees family=A region=22b286596054a5e4265c6f73c6acf0d1] 2024-11-08T00:26:35,456 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/1d77141220eb462d9bb2d82e6566176b is 175, key is test_row_0/A:col10/1731025595028/Put/seqid=0 2024-11-08T00:26:35,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742546_1722 (size=31255) 2024-11-08T00:26:35,463 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=437, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/1d77141220eb462d9bb2d82e6566176b 2024-11-08T00:26:35,473 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/e283ff1d3b954341bbb0f12721511690 is 50, key is test_row_0/B:col10/1731025595028/Put/seqid=0 2024-11-08T00:26:35,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742547_1723 (size=12301) 2024-11-08T00:26:35,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-08T00:26:35,879 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/e283ff1d3b954341bbb0f12721511690 2024-11-08T00:26:35,891 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/dd18a7bba92f4514a9a45c5969cf8d2c is 50, key is test_row_0/C:col10/1731025595028/Put/seqid=0 2024-11-08T00:26:35,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742548_1724 (size=12301) 2024-11-08T00:26:36,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-08T00:26:36,297 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/dd18a7bba92f4514a9a45c5969cf8d2c 2024-11-08T00:26:36,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/A/1d77141220eb462d9bb2d82e6566176b as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/1d77141220eb462d9bb2d82e6566176b 2024-11-08T00:26:36,307 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/1d77141220eb462d9bb2d82e6566176b, entries=150, sequenceid=437, filesize=30.5 K 2024-11-08T00:26:36,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/B/e283ff1d3b954341bbb0f12721511690 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/e283ff1d3b954341bbb0f12721511690 2024-11-08T00:26:36,310 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/e283ff1d3b954341bbb0f12721511690, entries=150, sequenceid=437, filesize=12.0 K 2024-11-08T00:26:36,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/.tmp/C/dd18a7bba92f4514a9a45c5969cf8d2c as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/dd18a7bba92f4514a9a45c5969cf8d2c 2024-11-08T00:26:36,314 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/dd18a7bba92f4514a9a45c5969cf8d2c, entries=150, sequenceid=437, filesize=12.0 K 2024-11-08T00:26:36,314 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=0 B/0 for 22b286596054a5e4265c6f73c6acf0d1 in 1283ms, sequenceid=437, compaction requested=false 2024-11-08T00:26:36,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 22b286596054a5e4265c6f73c6acf0d1: 2024-11-08T00:26:36,314 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:36,314 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:36,314 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. after waiting 0 ms 2024-11-08T00:26:36,314 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:36,315 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/ef2c8bb22d0e4516b4d7b56320df23ab, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/a283c9870e694513904e0905fcdeac13, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/a5f6c7f792a046489eeb2fe6708e2a44, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/232afd805d87439ea774f15fe285d6af, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/34a8ed2db6c64f5189c03977d5e79b10, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/1ed3ace8f08142a59b691b167cfed180, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/3209449326ae4d4296fc162069668e43, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/e7f12ca8eeab448c8201c7a3f14d6a4a, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/b7541ca9eb774e10ba90a134449f8b13, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/bbb35a1bcef647a3b87294f1a9db9cb1, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/92704af1b09344a6bd680916e527922e, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/fa7eea9f75824ba7b5acff453eb5ab6b, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/57a6b536c2a0456d94184a290230c2be, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/c383773730304b03bb1ab414ad85062b, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/b87654b6d07c49a488668354d98b3ef8, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/6d2d6c98262348229637d037f88d700f, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/b40f154deef04282a438b416dd15bbbc, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/54f2c2b88c9949a8924a8adc7f85ff29, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/000864556f4748668d3deb8a9ad42d96, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/6a227cc8a85b406ea3667e692ab65bf7, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/fc671ddb22a145a6ba5929fc0eedc7a8, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/0edbab4084c9499598678c26cb72e2c0, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/440f4b47c3d64f738eca8e80419088a6, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/b79df6fe14f74e7d95471dc2bbdad9a6, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/abdabb118a2a4c8cbceb378c9f8924dd, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/4d768e9836064c0eba2c7b3866ad8fc7, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/00f033d8aba1401ca73b89d5981b7427, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/5cf77d09123c4da7b3d398b043065f89, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/ff6d482b91304236a471afc439bb4dd8] to archive 2024-11-08T00:26:36,316 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T00:26:36,317 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/ef2c8bb22d0e4516b4d7b56320df23ab to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/ef2c8bb22d0e4516b4d7b56320df23ab 2024-11-08T00:26:36,318 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/a283c9870e694513904e0905fcdeac13 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/a283c9870e694513904e0905fcdeac13 2024-11-08T00:26:36,318 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/a5f6c7f792a046489eeb2fe6708e2a44 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/a5f6c7f792a046489eeb2fe6708e2a44 2024-11-08T00:26:36,319 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/232afd805d87439ea774f15fe285d6af to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/232afd805d87439ea774f15fe285d6af 2024-11-08T00:26:36,320 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/34a8ed2db6c64f5189c03977d5e79b10 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/34a8ed2db6c64f5189c03977d5e79b10 2024-11-08T00:26:36,321 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/1ed3ace8f08142a59b691b167cfed180 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/1ed3ace8f08142a59b691b167cfed180 2024-11-08T00:26:36,322 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/3209449326ae4d4296fc162069668e43 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/3209449326ae4d4296fc162069668e43 2024-11-08T00:26:36,323 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/e7f12ca8eeab448c8201c7a3f14d6a4a to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/e7f12ca8eeab448c8201c7a3f14d6a4a 2024-11-08T00:26:36,324 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/b7541ca9eb774e10ba90a134449f8b13 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/b7541ca9eb774e10ba90a134449f8b13 2024-11-08T00:26:36,325 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/bbb35a1bcef647a3b87294f1a9db9cb1 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/bbb35a1bcef647a3b87294f1a9db9cb1 2024-11-08T00:26:36,325 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/92704af1b09344a6bd680916e527922e to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/92704af1b09344a6bd680916e527922e 2024-11-08T00:26:36,326 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/fa7eea9f75824ba7b5acff453eb5ab6b to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/fa7eea9f75824ba7b5acff453eb5ab6b 2024-11-08T00:26:36,327 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/57a6b536c2a0456d94184a290230c2be to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/57a6b536c2a0456d94184a290230c2be 2024-11-08T00:26:36,328 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/c383773730304b03bb1ab414ad85062b to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/c383773730304b03bb1ab414ad85062b 2024-11-08T00:26:36,329 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/b87654b6d07c49a488668354d98b3ef8 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/b87654b6d07c49a488668354d98b3ef8 2024-11-08T00:26:36,330 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/6d2d6c98262348229637d037f88d700f to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/6d2d6c98262348229637d037f88d700f 2024-11-08T00:26:36,330 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/b40f154deef04282a438b416dd15bbbc to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/b40f154deef04282a438b416dd15bbbc 2024-11-08T00:26:36,331 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/54f2c2b88c9949a8924a8adc7f85ff29 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/54f2c2b88c9949a8924a8adc7f85ff29 2024-11-08T00:26:36,332 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/000864556f4748668d3deb8a9ad42d96 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/000864556f4748668d3deb8a9ad42d96 2024-11-08T00:26:36,333 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/6a227cc8a85b406ea3667e692ab65bf7 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/6a227cc8a85b406ea3667e692ab65bf7 2024-11-08T00:26:36,334 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/fc671ddb22a145a6ba5929fc0eedc7a8 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/fc671ddb22a145a6ba5929fc0eedc7a8 2024-11-08T00:26:36,334 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/0edbab4084c9499598678c26cb72e2c0 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/0edbab4084c9499598678c26cb72e2c0 2024-11-08T00:26:36,335 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/440f4b47c3d64f738eca8e80419088a6 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/440f4b47c3d64f738eca8e80419088a6 2024-11-08T00:26:36,336 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/b79df6fe14f74e7d95471dc2bbdad9a6 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/b79df6fe14f74e7d95471dc2bbdad9a6 2024-11-08T00:26:36,337 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/abdabb118a2a4c8cbceb378c9f8924dd to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/abdabb118a2a4c8cbceb378c9f8924dd 2024-11-08T00:26:36,337 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/4d768e9836064c0eba2c7b3866ad8fc7 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/4d768e9836064c0eba2c7b3866ad8fc7 2024-11-08T00:26:36,338 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/00f033d8aba1401ca73b89d5981b7427 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/00f033d8aba1401ca73b89d5981b7427 2024-11-08T00:26:36,339 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/5cf77d09123c4da7b3d398b043065f89 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/5cf77d09123c4da7b3d398b043065f89 2024-11-08T00:26:36,340 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/ff6d482b91304236a471afc439bb4dd8 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/ff6d482b91304236a471afc439bb4dd8 2024-11-08T00:26:36,341 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/ccdc5e2b52904f77a06aab690f01dc0c, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/a05055c84b88406c8ad1aa4d613d8b0d, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/f489aa1fd69e48ba82a900ec5709026e, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/70124e565243402ab7f0554adf229e8e, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/8463d10a3f5240c2bb6f1cd3f74d75a1, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/1e502dbf63ee4f74bb02901664c031c5, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/6fd7cbfede4a4f729edbe9e7a8e3dde0, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/8797895ce3d645b99b54dc7b6d8332fd, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/13b2b7cc0dc44fcaa7c435ac4237597a, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/057b5fb0f7a1462488af01d61cc0b618, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/74e17db025a74094ac3bcc0c1c0f3b1c, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/5f34c0ccbd4d4a198cdb184917d93dcc, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/091b7dae942f4398a2be115d3796aed7, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/050b5d5cbf62438fbac16daebdcd1a0a, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/810f21d192084919af9625a6dedf63fc, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/73b5f4b3ad87428a906d745e8e4edeae, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/7e8f0c114508427c9309f52155c7ee40, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/e02d9475bbfd4d46a54efebf474e7b24, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/1c8ff002dc2f4d6ea0605bc759da8f27, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/829365a4219a4b47977b2e8d6c50c014, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/824b72f7100645299f15f31b8e6f1450, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/23a604109da64446ae3066710e77cb8c, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/1cb82e6c9a024e8197550ba86c7b9bac, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/ff1156af08a5411b890669705e4326d5, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/4ef61c41d4f44741933da3240da4bc2f, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/f98494c77740482eb2e2be5975f27502, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/07778fe7495c4a23a651120ab5195f9b, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/f50b6009227945d095a75d0262983a84, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/c9e38f52b4554cd68e27c92d4617a778] to archive 2024-11-08T00:26:36,342 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T00:26:36,343 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/ccdc5e2b52904f77a06aab690f01dc0c to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/ccdc5e2b52904f77a06aab690f01dc0c 2024-11-08T00:26:36,344 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/a05055c84b88406c8ad1aa4d613d8b0d to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/a05055c84b88406c8ad1aa4d613d8b0d 2024-11-08T00:26:36,345 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/f489aa1fd69e48ba82a900ec5709026e to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/f489aa1fd69e48ba82a900ec5709026e 2024-11-08T00:26:36,346 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/70124e565243402ab7f0554adf229e8e to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/70124e565243402ab7f0554adf229e8e 2024-11-08T00:26:36,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/8463d10a3f5240c2bb6f1cd3f74d75a1 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/8463d10a3f5240c2bb6f1cd3f74d75a1 2024-11-08T00:26:36,348 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/1e502dbf63ee4f74bb02901664c031c5 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/1e502dbf63ee4f74bb02901664c031c5 2024-11-08T00:26:36,349 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/6fd7cbfede4a4f729edbe9e7a8e3dde0 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/6fd7cbfede4a4f729edbe9e7a8e3dde0 2024-11-08T00:26:36,350 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/8797895ce3d645b99b54dc7b6d8332fd to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/8797895ce3d645b99b54dc7b6d8332fd 2024-11-08T00:26:36,350 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/13b2b7cc0dc44fcaa7c435ac4237597a to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/13b2b7cc0dc44fcaa7c435ac4237597a 2024-11-08T00:26:36,351 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/057b5fb0f7a1462488af01d61cc0b618 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/057b5fb0f7a1462488af01d61cc0b618 2024-11-08T00:26:36,352 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/74e17db025a74094ac3bcc0c1c0f3b1c to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/74e17db025a74094ac3bcc0c1c0f3b1c 2024-11-08T00:26:36,353 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/5f34c0ccbd4d4a198cdb184917d93dcc to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/5f34c0ccbd4d4a198cdb184917d93dcc 2024-11-08T00:26:36,354 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/091b7dae942f4398a2be115d3796aed7 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/091b7dae942f4398a2be115d3796aed7 2024-11-08T00:26:36,354 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/050b5d5cbf62438fbac16daebdcd1a0a to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/050b5d5cbf62438fbac16daebdcd1a0a 2024-11-08T00:26:36,355 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/810f21d192084919af9625a6dedf63fc to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/810f21d192084919af9625a6dedf63fc 2024-11-08T00:26:36,356 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/73b5f4b3ad87428a906d745e8e4edeae to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/73b5f4b3ad87428a906d745e8e4edeae 2024-11-08T00:26:36,357 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/7e8f0c114508427c9309f52155c7ee40 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/7e8f0c114508427c9309f52155c7ee40 2024-11-08T00:26:36,357 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/e02d9475bbfd4d46a54efebf474e7b24 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/e02d9475bbfd4d46a54efebf474e7b24 2024-11-08T00:26:36,358 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/1c8ff002dc2f4d6ea0605bc759da8f27 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/1c8ff002dc2f4d6ea0605bc759da8f27 2024-11-08T00:26:36,359 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/829365a4219a4b47977b2e8d6c50c014 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/829365a4219a4b47977b2e8d6c50c014 2024-11-08T00:26:36,359 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/824b72f7100645299f15f31b8e6f1450 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/824b72f7100645299f15f31b8e6f1450 2024-11-08T00:26:36,360 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/23a604109da64446ae3066710e77cb8c to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/23a604109da64446ae3066710e77cb8c 2024-11-08T00:26:36,361 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/1cb82e6c9a024e8197550ba86c7b9bac to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/1cb82e6c9a024e8197550ba86c7b9bac 2024-11-08T00:26:36,362 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/ff1156af08a5411b890669705e4326d5 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/ff1156af08a5411b890669705e4326d5 2024-11-08T00:26:36,362 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/4ef61c41d4f44741933da3240da4bc2f to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/4ef61c41d4f44741933da3240da4bc2f 2024-11-08T00:26:36,363 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/f98494c77740482eb2e2be5975f27502 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/f98494c77740482eb2e2be5975f27502 2024-11-08T00:26:36,364 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/07778fe7495c4a23a651120ab5195f9b to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/07778fe7495c4a23a651120ab5195f9b 2024-11-08T00:26:36,364 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/f50b6009227945d095a75d0262983a84 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/f50b6009227945d095a75d0262983a84 2024-11-08T00:26:36,365 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/c9e38f52b4554cd68e27c92d4617a778 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/c9e38f52b4554cd68e27c92d4617a778 2024-11-08T00:26:36,366 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/fbc50f51b31d414d88ee9fb57e269475, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/806985b1f8b84264847d5ff29373995e, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/ad8a4112bbab49e4a9dff46ea980ffd7, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/a2a03013553b4b439489d4ccdbb24efa, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/9bf9df4a42ee4da5972d6b83c4da5f1a, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/8039654462934ed886a240d10506afc5, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/653e3ef9faff4b14b1278b534a548963, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/180a2b66a3ab46318244a474bd3f6b56, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/3c07df97f0c74d0fb599a5685a6877b9, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/3fcd612df23540c59b0063051f80248f, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/8b00792d0c0d4f16bad5dd9becf4e428, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/17dea555a5704a91a49a5864cc27eac1, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/665538b71af64fc9a3d9ace3896b7a3a, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/c32d97350ee943bb9f675314f961e94d, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/688a23bf04854367b7e7f0be6a1eff88, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/e76c092b0bd94d748a244cb6597d5fda, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/64be8aea7c3547b1b086e06604add343, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/9c6c926bd4f6441f868048e6a492648e, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/52b8c034c60c4de783fca133084ce0bd, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/ddd9a4ef90644ec4aab4e8e449c93cb3, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/03a46c21944a4b56af542ea48a43ced3, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/368e2e2b21154bc491d609bd195f55d7, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/db2b44e044af4e31a347b7cc89501de6, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/d64221850c75472ea7232b9e04aaefa8, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/ac29ad39024c4199b63050f849fc4d34, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/7fb91777fa024d298168e256720e44c7, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/81f7a1f6da8b48279f1816cb92e4f5be, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/b4e564b1feaa436083684cfb8bb8bd93] to archive 2024-11-08T00:26:36,367 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T00:26:36,368 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/fbc50f51b31d414d88ee9fb57e269475 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/fbc50f51b31d414d88ee9fb57e269475 2024-11-08T00:26:36,369 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/806985b1f8b84264847d5ff29373995e to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/806985b1f8b84264847d5ff29373995e 2024-11-08T00:26:36,369 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/ad8a4112bbab49e4a9dff46ea980ffd7 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/ad8a4112bbab49e4a9dff46ea980ffd7 2024-11-08T00:26:36,370 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/a2a03013553b4b439489d4ccdbb24efa to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/a2a03013553b4b439489d4ccdbb24efa 2024-11-08T00:26:36,371 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/9bf9df4a42ee4da5972d6b83c4da5f1a to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/9bf9df4a42ee4da5972d6b83c4da5f1a 2024-11-08T00:26:36,372 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/8039654462934ed886a240d10506afc5 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/8039654462934ed886a240d10506afc5 2024-11-08T00:26:36,372 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/653e3ef9faff4b14b1278b534a548963 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/653e3ef9faff4b14b1278b534a548963 2024-11-08T00:26:36,373 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/180a2b66a3ab46318244a474bd3f6b56 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/180a2b66a3ab46318244a474bd3f6b56 2024-11-08T00:26:36,373 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/3c07df97f0c74d0fb599a5685a6877b9 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/3c07df97f0c74d0fb599a5685a6877b9 2024-11-08T00:26:36,374 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/3fcd612df23540c59b0063051f80248f to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/3fcd612df23540c59b0063051f80248f 2024-11-08T00:26:36,375 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/8b00792d0c0d4f16bad5dd9becf4e428 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/8b00792d0c0d4f16bad5dd9becf4e428 2024-11-08T00:26:36,375 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/17dea555a5704a91a49a5864cc27eac1 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/17dea555a5704a91a49a5864cc27eac1 2024-11-08T00:26:36,376 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/665538b71af64fc9a3d9ace3896b7a3a to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/665538b71af64fc9a3d9ace3896b7a3a 2024-11-08T00:26:36,377 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/c32d97350ee943bb9f675314f961e94d to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/c32d97350ee943bb9f675314f961e94d 2024-11-08T00:26:36,378 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/688a23bf04854367b7e7f0be6a1eff88 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/688a23bf04854367b7e7f0be6a1eff88 2024-11-08T00:26:36,378 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/e76c092b0bd94d748a244cb6597d5fda to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/e76c092b0bd94d748a244cb6597d5fda 2024-11-08T00:26:36,379 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/64be8aea7c3547b1b086e06604add343 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/64be8aea7c3547b1b086e06604add343 2024-11-08T00:26:36,380 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/9c6c926bd4f6441f868048e6a492648e to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/9c6c926bd4f6441f868048e6a492648e 2024-11-08T00:26:36,381 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/52b8c034c60c4de783fca133084ce0bd to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/52b8c034c60c4de783fca133084ce0bd 2024-11-08T00:26:36,381 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/ddd9a4ef90644ec4aab4e8e449c93cb3 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/ddd9a4ef90644ec4aab4e8e449c93cb3 2024-11-08T00:26:36,382 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/03a46c21944a4b56af542ea48a43ced3 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/03a46c21944a4b56af542ea48a43ced3 2024-11-08T00:26:36,383 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/368e2e2b21154bc491d609bd195f55d7 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/368e2e2b21154bc491d609bd195f55d7 2024-11-08T00:26:36,384 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/db2b44e044af4e31a347b7cc89501de6 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/db2b44e044af4e31a347b7cc89501de6 2024-11-08T00:26:36,384 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/d64221850c75472ea7232b9e04aaefa8 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/d64221850c75472ea7232b9e04aaefa8 2024-11-08T00:26:36,385 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/ac29ad39024c4199b63050f849fc4d34 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/ac29ad39024c4199b63050f849fc4d34 2024-11-08T00:26:36,386 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/7fb91777fa024d298168e256720e44c7 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/7fb91777fa024d298168e256720e44c7 2024-11-08T00:26:36,387 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/81f7a1f6da8b48279f1816cb92e4f5be to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/81f7a1f6da8b48279f1816cb92e4f5be 2024-11-08T00:26:36,387 DEBUG [StoreCloser-TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/b4e564b1feaa436083684cfb8bb8bd93 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/b4e564b1feaa436083684cfb8bb8bd93 2024-11-08T00:26:36,391 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/recovered.edits/440.seqid, newMaxSeqId=440, maxSeqId=4 2024-11-08T00:26:36,391 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1. 2024-11-08T00:26:36,391 DEBUG [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1676): Region close journal for 22b286596054a5e4265c6f73c6acf0d1: Waiting for close lock at 1731025595199Running coprocessor pre-close hooks at 1731025595199Disabling compacts and flushes for region at 1731025595199Disabling writes for close at 1731025596314 (+1115 ms)Writing region close event to WAL at 1731025596389 (+75 ms)Running coprocessor post-close hooks at 1731025596391 (+2 ms)Closed at 1731025596391 2024-11-08T00:26:36,393 INFO [RS_CLOSE_REGION-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] handler.UnassignRegionHandler(157): Closed 22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:36,393 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=168 updating hbase:meta row=22b286596054a5e4265c6f73c6acf0d1, regionState=CLOSED 2024-11-08T00:26:36,395 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=169, ppid=168, state=RUNNABLE, hasLock=false; CloseRegionProcedure 22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 because future has completed 2024-11-08T00:26:36,396 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-08T00:26:36,397 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; CloseRegionProcedure 22b286596054a5e4265c6f73c6acf0d1, server=3302f0f507bd,40479,1731025415856 in 1.3490 sec 2024-11-08T00:26:36,398 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=168, resume processing ppid=167 2024-11-08T00:26:36,398 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, ppid=167, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=22b286596054a5e4265c6f73c6acf0d1, UNASSIGN in 1.3540 sec 2024-11-08T00:26:36,400 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-08T00:26:36,400 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.3580 sec 2024-11-08T00:26:36,402 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731025596402"}]},"ts":"1731025596402"} 2024-11-08T00:26:36,403 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-08T00:26:36,403 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-08T00:26:36,405 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.3680 sec 2024-11-08T00:26:37,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-08T00:26:37,198 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:26:37,198 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-08T00:26:37,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T00:26:37,200 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=170, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T00:26:37,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-08T00:26:37,200 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=170, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T00:26:37,201 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:37,203 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A, FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B, FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C, FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/recovered.edits] 2024-11-08T00:26:37,205 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/1d77141220eb462d9bb2d82e6566176b to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/1d77141220eb462d9bb2d82e6566176b 2024-11-08T00:26:37,206 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/3f907a9d688d49b993654ea4daa78891 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/A/3f907a9d688d49b993654ea4daa78891 2024-11-08T00:26:37,208 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/e283ff1d3b954341bbb0f12721511690 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/e283ff1d3b954341bbb0f12721511690 2024-11-08T00:26:37,209 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/e8426c72b16e4f1e847d795796eb191e to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/B/e8426c72b16e4f1e847d795796eb191e 2024-11-08T00:26:37,210 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/89588cade07d486a95e1695b8dad3b74 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/89588cade07d486a95e1695b8dad3b74 2024-11-08T00:26:37,210 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/dd18a7bba92f4514a9a45c5969cf8d2c to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/C/dd18a7bba92f4514a9a45c5969cf8d2c 2024-11-08T00:26:37,212 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/recovered.edits/440.seqid to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1/recovered.edits/440.seqid 2024-11-08T00:26:37,212 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/default/TestAcidGuarantees/22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:37,212 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-08T00:26:37,213 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-08T00:26:37,213 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-08T00:26:37,215 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411080237ef4db6de42ca97927ccb13e7dd0b_22b286596054a5e4265c6f73c6acf0d1 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411080237ef4db6de42ca97927ccb13e7dd0b_22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:37,216 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411080c6b074dd4b5469e939cbe22a0876c56_22b286596054a5e4265c6f73c6acf0d1 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411080c6b074dd4b5469e939cbe22a0876c56_22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:37,217 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411081a005640951a4e6689617bf6b555dc2d_22b286596054a5e4265c6f73c6acf0d1 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411081a005640951a4e6689617bf6b555dc2d_22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:37,218 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411082765f563d7174fcfac56d4843f1b5716_22b286596054a5e4265c6f73c6acf0d1 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411082765f563d7174fcfac56d4843f1b5716_22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:37,219 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411082b68ef4670374642b9fc5be0cf7a07d8_22b286596054a5e4265c6f73c6acf0d1 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411082b68ef4670374642b9fc5be0cf7a07d8_22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:37,220 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411084bd9e2b8f170478690c3970b277f1a17_22b286596054a5e4265c6f73c6acf0d1 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411084bd9e2b8f170478690c3970b277f1a17_22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:37,221 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411085e6e1853ba104019be4d18403ec02423_22b286596054a5e4265c6f73c6acf0d1 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411085e6e1853ba104019be4d18403ec02423_22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:37,221 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411085f29f7a8392a47cfaa57fca01bde52bc_22b286596054a5e4265c6f73c6acf0d1 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411085f29f7a8392a47cfaa57fca01bde52bc_22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:37,222 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108604b1612b67946a3827f0d129ddb9578_22b286596054a5e4265c6f73c6acf0d1 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108604b1612b67946a3827f0d129ddb9578_22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:37,223 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411086561f7f4c823487ea41595bf134e3246_22b286596054a5e4265c6f73c6acf0d1 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411086561f7f4c823487ea41595bf134e3246_22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:37,224 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110876930a9370eb4074a25753803aaed8a5_22b286596054a5e4265c6f73c6acf0d1 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110876930a9370eb4074a25753803aaed8a5_22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:37,224 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411087922c1c97305451ab0a94dd239304c5d_22b286596054a5e4265c6f73c6acf0d1 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411087922c1c97305451ab0a94dd239304c5d_22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:37,225 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411087b2ec87270ff49c8ac9c558a9a69795d_22b286596054a5e4265c6f73c6acf0d1 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411087b2ec87270ff49c8ac9c558a9a69795d_22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:37,226 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110881ad7bb350a94dafa259339fa3c89829_22b286596054a5e4265c6f73c6acf0d1 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110881ad7bb350a94dafa259339fa3c89829_22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:37,226 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108828fb5ef01394621854944824ba1b333_22b286596054a5e4265c6f73c6acf0d1 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108828fb5ef01394621854944824ba1b333_22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:37,227 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411089d604e62cf804fabb2872d3ea806a776_22b286596054a5e4265c6f73c6acf0d1 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411089d604e62cf804fabb2872d3ea806a776_22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:37,228 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108ad611dc453d44a8d8d6caf13202e6376_22b286596054a5e4265c6f73c6acf0d1 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108ad611dc453d44a8d8d6caf13202e6376_22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:37,228 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108adae130fabea4c168457d2b769ce6025_22b286596054a5e4265c6f73c6acf0d1 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108adae130fabea4c168457d2b769ce6025_22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:37,229 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108b90a64edc8c24e6e82a0c9058e2049b3_22b286596054a5e4265c6f73c6acf0d1 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108b90a64edc8c24e6e82a0c9058e2049b3_22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:37,230 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108d8b3bde183904f828e37633ceba247a6_22b286596054a5e4265c6f73c6acf0d1 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108d8b3bde183904f828e37633ceba247a6_22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:37,230 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108f5d9b60140c04c44939a3d9c9252b98d_22b286596054a5e4265c6f73c6acf0d1 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108f5d9b60140c04c44939a3d9c9252b98d_22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:37,231 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108fd88b2773fcf44eabdabb625caee29b3_22b286596054a5e4265c6f73c6acf0d1 to hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108fd88b2773fcf44eabdabb625caee29b3_22b286596054a5e4265c6f73c6acf0d1 2024-11-08T00:26:37,231 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-08T00:26:37,233 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=170, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T00:26:37,235 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-08T00:26:37,236 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-08T00:26:37,237 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=170, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T00:26:37,237 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-08T00:26:37,237 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731025597237"}]},"ts":"9223372036854775807"} 2024-11-08T00:26:37,239 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-08T00:26:37,239 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 22b286596054a5e4265c6f73c6acf0d1, NAME => 'TestAcidGuarantees,,1731025570014.22b286596054a5e4265c6f73c6acf0d1.', STARTKEY => '', ENDKEY => ''}] 2024-11-08T00:26:37,239 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-08T00:26:37,239 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731025597239"}]},"ts":"9223372036854775807"} 2024-11-08T00:26:37,240 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-08T00:26:37,241 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=170, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T00:26:37,242 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 43 msec 2024-11-08T00:26:37,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36785 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-08T00:26:37,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-08T00:26:37,308 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-08T00:26:37,322 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=241 (was 240) - Thread LEAK? -, OpenFileDescriptor=457 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=553 (was 591), ProcessCount=11 (was 11), AvailableMemoryMB=6264 (was 6385) 2024-11-08T00:26:37,323 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-08T00:26:37,323 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T00:26:37,323 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T00:26:37,323 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:26:37,323 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:26:37,323 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T00:26:37,323 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-08T00:26:37,323 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=741325824, stopped=false 2024-11-08T00:26:37,324 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=3302f0f507bd,36785,1731025414356 2024-11-08T00:26:37,383 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36785-0x10117d39f8d0000, quorum=127.0.0.1:49210, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T00:26:37,383 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36785-0x10117d39f8d0000, quorum=127.0.0.1:49210, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:26:37,383 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40479-0x10117d39f8d0001, quorum=127.0.0.1:49210, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T00:26:37,383 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T00:26:37,383 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40479-0x10117d39f8d0001, quorum=127.0.0.1:49210, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:26:37,384 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T00:26:37,384 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T00:26:37,384 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:26:37,385 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '3302f0f507bd,40479,1731025415856' ***** 2024-11-08T00:26:37,385 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40479-0x10117d39f8d0001, quorum=127.0.0.1:49210, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T00:26:37,386 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-08T00:26:37,386 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36785-0x10117d39f8d0000, quorum=127.0.0.1:49210, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T00:26:37,386 INFO [RS:0;3302f0f507bd:40479 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-08T00:26:37,387 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-08T00:26:37,387 INFO [RS:0;3302f0f507bd:40479 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-08T00:26:37,387 INFO [RS:0;3302f0f507bd:40479 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-08T00:26:37,387 INFO [RS:0;3302f0f507bd:40479 {}] regionserver.HRegionServer(959): stopping server 3302f0f507bd,40479,1731025415856 2024-11-08T00:26:37,387 INFO [RS:0;3302f0f507bd:40479 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T00:26:37,387 INFO [RS:0;3302f0f507bd:40479 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;3302f0f507bd:40479. 2024-11-08T00:26:37,388 DEBUG [RS:0;3302f0f507bd:40479 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T00:26:37,388 DEBUG [RS:0;3302f0f507bd:40479 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:26:37,388 INFO [RS:0;3302f0f507bd:40479 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-08T00:26:37,388 INFO [RS:0;3302f0f507bd:40479 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-08T00:26:37,388 INFO [RS:0;3302f0f507bd:40479 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-08T00:26:37,388 INFO [RS:0;3302f0f507bd:40479 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-08T00:26:37,389 INFO [RS:0;3302f0f507bd:40479 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-08T00:26:37,389 DEBUG [RS:0;3302f0f507bd:40479 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-08T00:26:37,389 DEBUG [RS:0;3302f0f507bd:40479 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-08T00:26:37,389 DEBUG [RS_CLOSE_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-08T00:26:37,390 INFO [RS_CLOSE_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-08T00:26:37,390 DEBUG [RS_CLOSE_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-08T00:26:37,390 DEBUG [RS_CLOSE_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-08T00:26:37,390 DEBUG [RS_CLOSE_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-08T00:26:37,390 INFO [RS_CLOSE_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-08T00:26:37,412 DEBUG [RS_CLOSE_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/hbase/meta/1588230740/.tmp/info/6b5629adbcd44e8ba2521f7f7956b615 is 95, key is TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691./info:/1731025450868/DeleteFamily/seqid=0 2024-11-08T00:26:37,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742549_1725 (size=6514) 2024-11-08T00:26:37,432 INFO [regionserver/3302f0f507bd:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T00:26:37,590 DEBUG [RS:0;3302f0f507bd:40479 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-08T00:26:37,611 INFO [regionserver/3302f0f507bd:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-08T00:26:37,611 INFO [regionserver/3302f0f507bd:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-08T00:26:37,790 DEBUG [RS:0;3302f0f507bd:40479 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-08T00:26:37,818 INFO [RS_CLOSE_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/hbase/meta/1588230740/.tmp/info/6b5629adbcd44e8ba2521f7f7956b615 2024-11-08T00:26:37,840 DEBUG [RS_CLOSE_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/hbase/meta/1588230740/.tmp/ns/9bcc749b0d1547b2b18a5d690a1e879a is 93, key is TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691./ns:/1731025450720/DeleteFamily/seqid=0 2024-11-08T00:26:37,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742550_1726 (size=5906) 2024-11-08T00:26:37,990 DEBUG [RS:0;3302f0f507bd:40479 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-08T00:26:38,190 DEBUG [RS:0;3302f0f507bd:40479 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-08T00:26:38,244 INFO [RS_CLOSE_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/hbase/meta/1588230740/.tmp/ns/9bcc749b0d1547b2b18a5d690a1e879a 2024-11-08T00:26:38,265 DEBUG [RS_CLOSE_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/hbase/meta/1588230740/.tmp/rep_barrier/55d49cd3611845ec97febbe4dd036277 is 102, key is TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691./rep_barrier:/1731025450720/DeleteFamily/seqid=0 2024-11-08T00:26:38,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742551_1727 (size=6025) 2024-11-08T00:26:38,390 DEBUG [RS:0;3302f0f507bd:40479 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-08T00:26:38,591 DEBUG [RS:0;3302f0f507bd:40479 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-08T00:26:38,668 INFO [RS_CLOSE_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/hbase/meta/1588230740/.tmp/rep_barrier/55d49cd3611845ec97febbe4dd036277 2024-11-08T00:26:38,685 DEBUG [RS_CLOSE_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/hbase/meta/1588230740/.tmp/table/7cbaab6d467e4c0b8e81c481410a95fc is 96, key is TestAcidGuarantees,,1731025420962.b211e3aadde87eeeb10e4f0dad149691./table:/1731025450720/DeleteFamily/seqid=0 2024-11-08T00:26:38,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742552_1728 (size=5936) 2024-11-08T00:26:38,791 DEBUG [RS:0;3302f0f507bd:40479 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-08T00:26:38,991 DEBUG [RS:0;3302f0f507bd:40479 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-08T00:26:39,088 INFO [RS_CLOSE_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/hbase/meta/1588230740/.tmp/table/7cbaab6d467e4c0b8e81c481410a95fc 2024-11-08T00:26:39,092 DEBUG [RS_CLOSE_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/hbase/meta/1588230740/.tmp/info/6b5629adbcd44e8ba2521f7f7956b615 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/hbase/meta/1588230740/info/6b5629adbcd44e8ba2521f7f7956b615 2024-11-08T00:26:39,094 INFO [RS_CLOSE_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/hbase/meta/1588230740/info/6b5629adbcd44e8ba2521f7f7956b615, entries=12, sequenceid=90, filesize=6.4 K 2024-11-08T00:26:39,095 DEBUG [RS_CLOSE_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/hbase/meta/1588230740/.tmp/ns/9bcc749b0d1547b2b18a5d690a1e879a as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/hbase/meta/1588230740/ns/9bcc749b0d1547b2b18a5d690a1e879a 2024-11-08T00:26:39,098 INFO [RS_CLOSE_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/hbase/meta/1588230740/ns/9bcc749b0d1547b2b18a5d690a1e879a, entries=8, sequenceid=90, filesize=5.8 K 2024-11-08T00:26:39,098 DEBUG [RS_CLOSE_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/hbase/meta/1588230740/.tmp/rep_barrier/55d49cd3611845ec97febbe4dd036277 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/hbase/meta/1588230740/rep_barrier/55d49cd3611845ec97febbe4dd036277 2024-11-08T00:26:39,101 INFO [RS_CLOSE_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/hbase/meta/1588230740/rep_barrier/55d49cd3611845ec97febbe4dd036277, entries=6, sequenceid=90, filesize=5.9 K 2024-11-08T00:26:39,101 DEBUG [RS_CLOSE_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/hbase/meta/1588230740/.tmp/table/7cbaab6d467e4c0b8e81c481410a95fc as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/hbase/meta/1588230740/table/7cbaab6d467e4c0b8e81c481410a95fc 2024-11-08T00:26:39,103 INFO [RS_CLOSE_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/hbase/meta/1588230740/table/7cbaab6d467e4c0b8e81c481410a95fc, entries=7, sequenceid=90, filesize=5.8 K 2024-11-08T00:26:39,104 INFO [RS_CLOSE_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1714ms, sequenceid=90, compaction requested=false 2024-11-08T00:26:39,107 DEBUG [RS_CLOSE_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-08T00:26:39,108 DEBUG [RS_CLOSE_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-08T00:26:39,108 INFO [RS_CLOSE_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-08T00:26:39,108 DEBUG [RS_CLOSE_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731025597389Running coprocessor pre-close hooks at 1731025597389Disabling compacts and flushes for region at 1731025597389Disabling writes for close at 1731025597390 (+1 ms)Obtaining lock to block concurrent updates at 1731025597390Preparing flush snapshotting stores in 1588230740 at 1731025597390Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731025597390Flushing stores of hbase:meta,,1.1588230740 at 1731025597391 (+1 ms)Flushing 1588230740/info: creating writer at 1731025597391Flushing 1588230740/info: appending metadata at 1731025597411 (+20 ms)Flushing 1588230740/info: closing flushed file at 1731025597411Flushing 1588230740/ns: creating writer at 1731025597821 (+410 ms)Flushing 1588230740/ns: appending metadata at 1731025597840 (+19 ms)Flushing 1588230740/ns: closing flushed file at 1731025597840Flushing 1588230740/rep_barrier: creating writer at 1731025598248 (+408 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731025598264 (+16 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731025598264Flushing 1588230740/table: creating writer at 1731025598671 (+407 ms)Flushing 1588230740/table: appending metadata at 1731025598685 (+14 ms)Flushing 1588230740/table: closing flushed file at 1731025598685Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7ffe63e5: reopening flushed file at 1731025599091 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a6c1c80: reopening flushed file at 1731025599095 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6407e8b8: reopening flushed file at 1731025599098 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@46c5cf57: reopening flushed file at 1731025599101 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1714ms, sequenceid=90, compaction requested=false at 1731025599104 (+3 ms)Writing region close event to WAL at 1731025599105 (+1 ms)Running coprocessor post-close hooks at 1731025599108 (+3 ms)Closed at 1731025599108 2024-11-08T00:26:39,108 DEBUG [RS_CLOSE_META-regionserver/3302f0f507bd:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-08T00:26:39,191 INFO [RS:0;3302f0f507bd:40479 {}] regionserver.HRegionServer(976): stopping server 3302f0f507bd,40479,1731025415856; all regions closed. 2024-11-08T00:26:39,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741834_1010 (size=25344) 2024-11-08T00:26:39,197 DEBUG [RS:0;3302f0f507bd:40479 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/oldWALs 2024-11-08T00:26:39,198 INFO [RS:0;3302f0f507bd:40479 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 3302f0f507bd%2C40479%2C1731025415856.meta:.meta(num 1731025420252) 2024-11-08T00:26:39,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741833_1009 (size=16497851) 2024-11-08T00:26:39,201 DEBUG [RS:0;3302f0f507bd:40479 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/oldWALs 2024-11-08T00:26:39,201 INFO [RS:0;3302f0f507bd:40479 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 3302f0f507bd%2C40479%2C1731025415856:(num 1731025419840) 2024-11-08T00:26:39,201 DEBUG [RS:0;3302f0f507bd:40479 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T00:26:39,201 INFO [RS:0;3302f0f507bd:40479 {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T00:26:39,201 INFO [RS:0;3302f0f507bd:40479 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T00:26:39,202 INFO [RS:0;3302f0f507bd:40479 {}] hbase.ChoreService(370): Chore service for: regionserver/3302f0f507bd:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-08T00:26:39,202 INFO [RS:0;3302f0f507bd:40479 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T00:26:39,202 INFO [regionserver/3302f0f507bd:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T00:26:39,202 INFO [RS:0;3302f0f507bd:40479 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:40479 2024-11-08T00:26:39,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36785-0x10117d39f8d0000, quorum=127.0.0.1:49210, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T00:26:39,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40479-0x10117d39f8d0001, quorum=127.0.0.1:49210, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/3302f0f507bd,40479,1731025415856 2024-11-08T00:26:39,257 INFO [RS:0;3302f0f507bd:40479 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T00:26:39,258 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [3302f0f507bd,40479,1731025415856] 2024-11-08T00:26:39,369 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/3302f0f507bd,40479,1731025415856 already deleted, retry=false 2024-11-08T00:26:39,370 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 3302f0f507bd,40479,1731025415856 expired; onlineServers=0 2024-11-08T00:26:39,370 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '3302f0f507bd,36785,1731025414356' ***** 2024-11-08T00:26:39,370 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-08T00:26:39,370 INFO [M:0;3302f0f507bd:36785 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T00:26:39,370 INFO [M:0;3302f0f507bd:36785 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T00:26:39,371 DEBUG [M:0;3302f0f507bd:36785 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-08T00:26:39,371 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-08T00:26:39,371 DEBUG [master/3302f0f507bd:0:becomeActiveMaster-HFileCleaner.small.0-1731025419244 {}] cleaner.HFileCleaner(306): Exit Thread[master/3302f0f507bd:0:becomeActiveMaster-HFileCleaner.small.0-1731025419244,5,FailOnTimeoutGroup] 2024-11-08T00:26:39,371 DEBUG [master/3302f0f507bd:0:becomeActiveMaster-HFileCleaner.large.0-1731025419243 {}] cleaner.HFileCleaner(306): Exit Thread[master/3302f0f507bd:0:becomeActiveMaster-HFileCleaner.large.0-1731025419243,5,FailOnTimeoutGroup] 2024-11-08T00:26:39,371 DEBUG [M:0;3302f0f507bd:36785 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-08T00:26:39,372 INFO [M:0;3302f0f507bd:36785 {}] hbase.ChoreService(370): Chore service for: master/3302f0f507bd:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-08T00:26:39,372 INFO [M:0;3302f0f507bd:36785 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T00:26:39,372 DEBUG [M:0;3302f0f507bd:36785 {}] master.HMaster(1795): Stopping service threads 2024-11-08T00:26:39,372 INFO [M:0;3302f0f507bd:36785 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-08T00:26:39,372 INFO [M:0;3302f0f507bd:36785 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T00:26:39,372 ERROR [M:0;3302f0f507bd:36785 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-5,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-08T00:26:39,373 INFO [M:0;3302f0f507bd:36785 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T00:26:39,374 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-08T00:26:39,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40479-0x10117d39f8d0001, quorum=127.0.0.1:49210, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T00:26:39,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40479-0x10117d39f8d0001, quorum=127.0.0.1:49210, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T00:26:39,408 INFO [RS:0;3302f0f507bd:40479 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T00:26:39,408 INFO [RS:0;3302f0f507bd:40479 {}] regionserver.HRegionServer(1031): Exiting; stopping=3302f0f507bd,40479,1731025415856; zookeeper connection closed. 2024-11-08T00:26:39,409 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4f67e358 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4f67e358 2024-11-08T00:26:39,409 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-08T00:26:39,423 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36785-0x10117d39f8d0000, quorum=127.0.0.1:49210, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-08T00:26:39,423 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36785-0x10117d39f8d0000, quorum=127.0.0.1:49210, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T00:26:39,423 DEBUG [M:0;3302f0f507bd:36785 {}] zookeeper.ZKUtil(347): master:36785-0x10117d39f8d0000, quorum=127.0.0.1:49210, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-08T00:26:39,423 WARN [M:0;3302f0f507bd:36785 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-08T00:26:39,424 INFO [M:0;3302f0f507bd:36785 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/.lastflushedseqids 2024-11-08T00:26:39,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742553_1729 (size=72) 2024-11-08T00:26:39,832 INFO [M:0;3302f0f507bd:36785 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T00:26:39,832 INFO [M:0;3302f0f507bd:36785 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-08T00:26:39,832 DEBUG [M:0;3302f0f507bd:36785 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-08T00:26:39,832 INFO [M:0;3302f0f507bd:36785 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T00:26:39,832 DEBUG [M:0;3302f0f507bd:36785 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T00:26:39,833 DEBUG [M:0;3302f0f507bd:36785 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 1 ms 2024-11-08T00:26:39,833 DEBUG [M:0;3302f0f507bd:36785 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T00:26:39,833 INFO [M:0;3302f0f507bd:36785 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=779.99 KB heapSize=959.89 KB 2024-11-08T00:26:39,847 DEBUG [M:0;3302f0f507bd:36785 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3009c7c8605b4907ad82f01bb15a6b22 is 82, key is hbase:meta,,1/info:regioninfo/1731025420405/Put/seqid=0 2024-11-08T00:26:39,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742554_1730 (size=5672) 2024-11-08T00:26:40,251 INFO [M:0;3302f0f507bd:36785 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2228 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3009c7c8605b4907ad82f01bb15a6b22 2024-11-08T00:26:40,277 DEBUG [M:0;3302f0f507bd:36785 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e06e12b9acb24c04ab591f63f9948813 is 2277, key is \x00\x00\x00\x00\x00\x00\x00\x93/proc:d/1731025573087/Put/seqid=0 2024-11-08T00:26:40,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742555_1731 (size=44206) 2024-11-08T00:26:40,682 INFO [M:0;3302f0f507bd:36785 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=779.44 KB at sequenceid=2228 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e06e12b9acb24c04ab591f63f9948813 2024-11-08T00:26:40,685 INFO [M:0;3302f0f507bd:36785 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e06e12b9acb24c04ab591f63f9948813 2024-11-08T00:26:40,700 DEBUG [M:0;3302f0f507bd:36785 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/57b6001736d4425e9793b87d49b866ab is 69, key is 3302f0f507bd,40479,1731025415856/rs:state/1731025419345/Put/seqid=0 2024-11-08T00:26:40,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073742556_1732 (size=5156) 2024-11-08T00:26:41,103 INFO [M:0;3302f0f507bd:36785 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2228 (bloomFilter=true), to=hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/57b6001736d4425e9793b87d49b866ab 2024-11-08T00:26:41,107 DEBUG [M:0;3302f0f507bd:36785 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3009c7c8605b4907ad82f01bb15a6b22 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/3009c7c8605b4907ad82f01bb15a6b22 2024-11-08T00:26:41,109 INFO [M:0;3302f0f507bd:36785 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/3009c7c8605b4907ad82f01bb15a6b22, entries=8, sequenceid=2228, filesize=5.5 K 2024-11-08T00:26:41,109 DEBUG [M:0;3302f0f507bd:36785 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e06e12b9acb24c04ab591f63f9948813 as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e06e12b9acb24c04ab591f63f9948813 2024-11-08T00:26:41,112 INFO [M:0;3302f0f507bd:36785 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e06e12b9acb24c04ab591f63f9948813 2024-11-08T00:26:41,112 INFO [M:0;3302f0f507bd:36785 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e06e12b9acb24c04ab591f63f9948813, entries=170, sequenceid=2228, filesize=43.2 K 2024-11-08T00:26:41,113 DEBUG [M:0;3302f0f507bd:36785 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/57b6001736d4425e9793b87d49b866ab as hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/57b6001736d4425e9793b87d49b866ab 2024-11-08T00:26:41,115 INFO [M:0;3302f0f507bd:36785 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34031/user/jenkins/test-data/5a3b4dae-724f-1842-5307-6857b45f661f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/57b6001736d4425e9793b87d49b866ab, entries=1, sequenceid=2228, filesize=5.0 K 2024-11-08T00:26:41,116 INFO [M:0;3302f0f507bd:36785 {}] regionserver.HRegion(3140): Finished flush of dataSize ~779.99 KB/798712, heapSize ~959.59 KB/982624, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1283ms, sequenceid=2228, compaction requested=false 2024-11-08T00:26:41,117 INFO [M:0;3302f0f507bd:36785 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T00:26:41,117 DEBUG [M:0;3302f0f507bd:36785 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731025599832Disabling compacts and flushes for region at 1731025599832Disabling writes for close at 1731025599833 (+1 ms)Obtaining lock to block concurrent updates at 1731025599833Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731025599833Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=798712, getHeapSize=982864, getOffHeapSize=0, getCellsCount=2423 at 1731025599833Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731025599834 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731025599834Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731025599847 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731025599847Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731025600258 (+411 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731025600277 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731025600277Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731025600685 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731025600699 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731025600699Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37bc3e47: reopening flushed file at 1731025601106 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7e20edbd: reopening flushed file at 1731025601109 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6d22ff9a: reopening flushed file at 1731025601112 (+3 ms)Finished flush of dataSize ~779.99 KB/798712, heapSize ~959.59 KB/982624, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1283ms, sequenceid=2228, compaction requested=false at 1731025601116 (+4 ms)Writing region close event to WAL at 1731025601117 (+1 ms)Closed at 1731025601117 2024-11-08T00:26:41,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39671 is added to blk_1073741830_1006 (size=944731) 2024-11-08T00:26:41,120 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T00:26:41,120 INFO [M:0;3302f0f507bd:36785 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-08T00:26:41,120 INFO [M:0;3302f0f507bd:36785 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:36785 2024-11-08T00:26:41,120 INFO [M:0;3302f0f507bd:36785 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T00:26:41,282 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36785-0x10117d39f8d0000, quorum=127.0.0.1:49210, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T00:26:41,282 INFO [M:0;3302f0f507bd:36785 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T00:26:41,282 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36785-0x10117d39f8d0000, quorum=127.0.0.1:49210, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T00:26:41,287 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@37e9dd02{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-08T00:26:41,290 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@8110c28{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-08T00:26:41,290 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-08T00:26:41,290 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3815836d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-08T00:26:41,290 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7a38c5d7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c8e766d-da5e-64ca-5ccd-0c91fcfb6de3/hadoop.log.dir/,STOPPED} 2024-11-08T00:26:41,294 WARN [BP-1018477837-172.17.0.3-1731025408374 heartbeating to localhost/127.0.0.1:34031 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-08T00:26:41,294 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-08T00:26:41,294 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-08T00:26:41,294 WARN [BP-1018477837-172.17.0.3-1731025408374 heartbeating to localhost/127.0.0.1:34031 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1018477837-172.17.0.3-1731025408374 (Datanode Uuid 6000dce3-f1e3-464a-a17d-199c856fc814) service to localhost/127.0.0.1:34031 2024-11-08T00:26:41,296 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c8e766d-da5e-64ca-5ccd-0c91fcfb6de3/cluster_72fb0791-5876-a833-2fa2-61645688b81f/data/data1/current/BP-1018477837-172.17.0.3-1731025408374 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-08T00:26:41,296 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c8e766d-da5e-64ca-5ccd-0c91fcfb6de3/cluster_72fb0791-5876-a833-2fa2-61645688b81f/data/data2/current/BP-1018477837-172.17.0.3-1731025408374 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-08T00:26:41,297 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-08T00:26:41,303 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7f405bdd{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-08T00:26:41,304 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5b829fcb{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-08T00:26:41,304 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-08T00:26:41,304 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@31a6a643{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-08T00:26:41,304 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@610d6b27{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2c8e766d-da5e-64ca-5ccd-0c91fcfb6de3/hadoop.log.dir/,STOPPED} 2024-11-08T00:26:41,318 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-08T00:26:41,393 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-08T00:26:41,532 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down